1/*
2Copyright 2018 Google Inc. All Rights Reserved.
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS-IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/
16
17#include "graph/binaural_surround_renderer_impl.h"
18
19#include <algorithm>
20#include <functional>
21
22#include "base/misc_math.h"
23#include "base/simd_utils.h"
24#include "base/spherical_angle.h"
25#include "graph/resonance_audio_api_impl.h"
26#include "platforms/common/room_effects_utils.h"
27#include "platforms/common/room_properties.h"
28#include "utils/planar_interleaved_conversion.h"
29
30namespace vraudio {
31
32namespace {
33
34// Maximum number of audio buffers in buffer queue.
35const size_t kNumMaxBuffers = 64;
36
37// Output gain, to avoid clipping of individual virtual speaker channels.
38const float kGain = 0.5f;
39
40} // namespace
41
42BinauralSurroundRendererImpl::BinauralSurroundRendererImpl(
43 size_t frames_per_buffer, int sample_rate_hz)
44 :
45 resonance_audio_api_(nullptr),
46 frames_per_buffer_(frames_per_buffer),
47 sample_rate_hz_(sample_rate_hz),
48 surround_format_(kInvalid),
49 num_input_channels_(0),
50 output_buffer_(kNumStereoChannels, frames_per_buffer),
51 total_frames_buffered_(0),
52 num_zero_padded_frames_(0),
53 output_gain_(1.0f) {
54}
55
56bool BinauralSurroundRendererImpl::Init(SurroundFormat surround_format) {
57 surround_format_ = surround_format;
58 num_input_channels_ =
59 GetExpectedNumChannelsFromSurroundFormat(surround_format);
60
61 temp_planar_buffer_ptrs_.resize(new_size: num_input_channels_);
62
63 input_audio_buffer_queue_.reset(p: new ThreadsafeFifo<AudioBuffer>(
64 kNumMaxBuffers, AudioBuffer(num_input_channels_, frames_per_buffer_)));
65
66 buffer_partitioner_.reset(p: new BufferPartitioner(
67 num_input_channels_, frames_per_buffer_,
68 std::bind(f: &BinauralSurroundRendererImpl::BufferPartitionerCallback, args: this,
69 args: std::placeholders::_1)));
70
71 buffer_unpartitioner_.reset(p: new BufferUnpartitioner(
72 kNumStereoChannels, frames_per_buffer_,
73 std::bind(f: &BinauralSurroundRendererImpl::ProcessBuffer, args: this)));
74
75 resonance_audio_api_.reset(p: CreateResonanceAudioApi(
76 num_channels: kNumStereoChannels, frames_per_buffer: frames_per_buffer_, sample_rate_hz: sample_rate_hz_));
77
78 if (surround_format == kSurroundMono || surround_format == kSurroundStereo ||
79 surround_format == kSurroundFiveDotOne ||
80 surround_format == kSurroundSevenDotOne) {
81 InitializeRoomReverb();
82 }
83 // Initialize rendering mode.
84 switch (surround_format) {
85 case kSurroundMono:
86 InitializeBinauralMono();
87 break;
88 case kSurroundStereo:
89 InitializeBinauralStereo();
90 break;
91 case kSurroundFiveDotOne:
92 InitializeBinauralSurround5dot1();
93 break;
94 case kSurroundSevenDotOne:
95 InitializeBinauralSurround7dot1();
96 break;
97 case kFirstOrderAmbisonics:
98 case kSecondOrderAmbisonics:
99 case kThirdOrderAmbisonics:
100 InitializeAmbisonics();
101 break;
102 case kFirstOrderAmbisonicsWithNonDiegeticStereo:
103 case kSecondOrderAmbisonicsWithNonDiegeticStereo:
104 case kThirdOrderAmbisonicsWithNonDiegeticStereo:
105 InitializeAmbisonicsWithNonDiegeticStereo();
106 break;
107 default:
108 LOG(FATAL) << "Undefined rendering mode";
109 return false;
110 break;
111 }
112 return true;
113}
114
115BinauralSurroundRendererImpl::BinauralSurroundRendererImpl()
116 :
117 resonance_audio_api_(nullptr),
118 frames_per_buffer_(0),
119 sample_rate_hz_(0),
120 total_frames_buffered_(0),
121 num_zero_padded_frames_(0) {
122}
123
124AudioBuffer* BinauralSurroundRendererImpl::BufferPartitionerCallback(
125 AudioBuffer* processed_buffer) {
126 if (processed_buffer != nullptr) {
127 input_audio_buffer_queue_->ReleaseInputObject(object: processed_buffer);
128 }
129 DCHECK(!input_audio_buffer_queue_->Full());
130 return input_audio_buffer_queue_->AcquireInputObject();
131}
132
133void BinauralSurroundRendererImpl::SetStereoSpeakerMode(bool enabled) {
134 resonance_audio_api_->SetStereoSpeakerMode(enabled);
135}
136
137size_t BinauralSurroundRendererImpl::GetExpectedNumChannelsFromSurroundFormat(
138 SurroundFormat surround_format) {
139 switch (surround_format) {
140 case kSurroundMono:
141 return kNumMonoChannels;
142 case kSurroundStereo:
143 return kNumStereoChannels;
144 case kSurroundFiveDotOne:
145 return kNumSurroundFiveDotOneChannels;
146 case kSurroundSevenDotOne:
147 return kNumSurroundSevenDotOneChannels;
148 case kFirstOrderAmbisonics:
149 return kNumFirstOrderAmbisonicChannels;
150 case kSecondOrderAmbisonics:
151 return kNumSecondOrderAmbisonicChannels;
152 case kThirdOrderAmbisonics:
153 return kNumThirdOrderAmbisonicChannels;
154 case kFirstOrderAmbisonicsWithNonDiegeticStereo:
155 return kNumFirstOrderAmbisonicChannels + kNumStereoChannels;
156 case kSecondOrderAmbisonicsWithNonDiegeticStereo:
157 return kNumSecondOrderAmbisonicChannels + kNumStereoChannels;
158 case kThirdOrderAmbisonicsWithNonDiegeticStereo:
159 return kNumThirdOrderAmbisonicChannels + kNumStereoChannels;
160 default:
161 LOG(FATAL) << "Undefined surround format mode";
162 return false;
163 break;
164 }
165 return 0;
166}
167
168void BinauralSurroundRendererImpl::InitializeBinauralMono() {
169 source_ids_.resize(new_size: kNumMonoChannels);
170 // Front (0 degrees):
171 source_ids_[0] = CreateSoundObject(azimuth_deg: 0.0f);
172 output_gain_ = kGain;
173}
174
175void BinauralSurroundRendererImpl::InitializeBinauralStereo() {
176
177 source_ids_.resize(new_size: kNumStereoChannels);
178 // Front left (30 degrees):
179 source_ids_[0] = CreateSoundObject(azimuth_deg: 30.0f);
180 // Front right (-30 degrees):
181 source_ids_[1] = CreateSoundObject(azimuth_deg: -30.0f);
182 output_gain_ = kGain;
183}
184
185void BinauralSurroundRendererImpl::InitializeBinauralSurround5dot1() {
186 source_ids_.resize(new_size: kNumSurroundFiveDotOneChannels);
187 // Left (30 degrees):
188 source_ids_[0] = CreateSoundObject(azimuth_deg: 30.0f);
189 // Right (-30 degrees):
190 source_ids_[1] = CreateSoundObject(azimuth_deg: -30.0f);
191 // Center (0 degrees):
192 source_ids_[2] = CreateSoundObject(azimuth_deg: 0.0f);
193 // Low frequency effects at front center:
194 source_ids_[3] = CreateSoundObject(azimuth_deg: 0.0f);
195 // Left surround (110 degrees):
196 source_ids_[4] = CreateSoundObject(azimuth_deg: 110.0f);
197 // Right surround (-110 degrees):
198 source_ids_[5] = CreateSoundObject(azimuth_deg: -110.0f);
199 output_gain_ = kGain;
200}
201
202void BinauralSurroundRendererImpl::InitializeBinauralSurround7dot1() {
203 source_ids_.resize(new_size: kNumSurroundSevenDotOneChannels);
204 // Left (30 degrees):
205 source_ids_[0] = CreateSoundObject(azimuth_deg: 30.0f);
206 // Right (-30 degrees):
207 source_ids_[1] = CreateSoundObject(azimuth_deg: -30.0f);
208 // Center (0 degrees):
209 source_ids_[2] = CreateSoundObject(azimuth_deg: 0.0f);
210 // Low frequency effects at front center:
211 source_ids_[3] = CreateSoundObject(azimuth_deg: 0.0f);
212 // Left surround 1 (90 degrees):
213 source_ids_[4] = CreateSoundObject(azimuth_deg: 90.0f);
214 // Right surround 1 (-90 degrees):
215 source_ids_[5] = CreateSoundObject(azimuth_deg: -90.0f);
216 // Left surround 2 (150 degrees):
217 source_ids_[6] = CreateSoundObject(azimuth_deg: 150.0f);
218 // Right surround 2 (-150 degrees):
219 source_ids_[7] = CreateSoundObject(azimuth_deg: -150.0f);
220 output_gain_ = kGain;
221}
222
223void BinauralSurroundRendererImpl::InitializeAmbisonics() {
224 source_ids_.resize(new_size: 1);
225 source_ids_[0] =
226 resonance_audio_api_->CreateAmbisonicSource(num_channels: num_input_channels_);
227}
228
229void BinauralSurroundRendererImpl::InitializeAmbisonicsWithNonDiegeticStereo() {
230 source_ids_.resize(new_size: 2);
231 CHECK_GT(num_input_channels_, kNumStereoChannels);
232 source_ids_[0] = resonance_audio_api_->CreateAmbisonicSource(
233 num_channels: num_input_channels_ - kNumStereoChannels);
234 source_ids_[1] = resonance_audio_api_->CreateStereoSource(num_channels: kNumStereoChannels);
235}
236
237SourceId BinauralSurroundRendererImpl::CreateSoundObject(float azimuth_deg) {
238 static const float kZeroElevation = 0.0f;
239 auto speaker_position =
240 vraudio::SphericalAngle::FromDegrees(azimuth_degrees: azimuth_deg, elevation_degrees: kZeroElevation)
241 .GetWorldPositionOnUnitSphere();
242 const SourceId source_id = resonance_audio_api_->CreateSoundObjectSource(
243 rendering_mode: RenderingMode::kBinauralHighQuality);
244 resonance_audio_api_->SetSourcePosition(
245 source_id, x: speaker_position[0], y: speaker_position[1], z: speaker_position[2]);
246 return source_id;
247}
248
249void BinauralSurroundRendererImpl::InitializeRoomReverb() {
250 // The following settings has been applied based on AESTD1001.1.01-10.
251 RoomProperties room_properties;
252 room_properties.dimensions[0] = 9.54f;
253 room_properties.dimensions[1] = 6.0f;
254 room_properties.dimensions[2] = 15.12f;
255 room_properties.reverb_brightness = 0.0f;
256 room_properties.reflection_scalar = 1.0f;
257 // Reduce reverb gain to compensate for virtual speakers gain.
258 room_properties.reverb_gain = output_gain_;
259 for (size_t i = 0; i < kNumRoomSurfaces; ++i) {
260 room_properties.material_names[i] = MaterialName::kUniform;
261 }
262 resonance_audio_api_->SetReflectionProperties(
263 ComputeReflectionProperties(room_properties));
264 resonance_audio_api_->SetReverbProperties(
265 ComputeReverbProperties(room_properties));
266 resonance_audio_api_->EnableRoomEffects(enable: true);
267}
268
269size_t BinauralSurroundRendererImpl::GetNumAvailableFramesInInputBuffer()
270 const {
271 DCHECK_NE(surround_format_, kInvalid);
272 if (num_zero_padded_frames_ > 0) {
273 // Zero padded output buffers must be consumed prior to
274 // |AddInterleavedBuffer| calls;
275 return 0;
276 }
277 if (input_audio_buffer_queue_->Full()) {
278 return 0;
279 }
280 // Subtract two buffers from the available input slots to ensure the buffer
281 // partitioner can be flushed at any time while keeping an extra buffer
282 // available in the |buffer_partitioner_| callback for the next incoming data.
283 const size_t num_frames_available_in_input_slots =
284 (kNumMaxBuffers - input_audio_buffer_queue_->Size() - 2) *
285 frames_per_buffer_;
286 DCHECK_GT(frames_per_buffer_, buffer_partitioner_->GetNumBufferedFrames());
287 const size_t num_frames_available_in_buffer_partitioner =
288 frames_per_buffer_ - buffer_partitioner_->GetNumBufferedFrames();
289 return num_frames_available_in_input_slots +
290 num_frames_available_in_buffer_partitioner;
291}
292
293size_t BinauralSurroundRendererImpl::AddInterleavedInput(
294 const int16* input_buffer_ptr, size_t num_channels, size_t num_frames) {
295 return AddInputBufferTemplated<const int16*>(input_buffer_ptr, num_channels,
296 num_frames);
297}
298
299size_t BinauralSurroundRendererImpl::AddInterleavedInput(
300 const float* input_buffer_ptr, size_t num_channels, size_t num_frames) {
301 return AddInputBufferTemplated<const float*>(input_buffer_ptr, num_channels,
302 num_frames);
303}
304
305size_t BinauralSurroundRendererImpl::AddPlanarInput(
306 const int16* const* input_buffer_ptrs, size_t num_channels,
307 size_t num_frames) {
308 return AddInputBufferTemplated<const int16* const*>(input_buffer_ptr: input_buffer_ptrs,
309 num_channels, num_frames);
310}
311
312size_t BinauralSurroundRendererImpl::AddPlanarInput(
313 const float* const* input_buffer_ptrs, size_t num_channels,
314 size_t num_frames) {
315 return AddInputBufferTemplated<const float* const*>(input_buffer_ptr: input_buffer_ptrs,
316 num_channels, num_frames);
317}
318
319template <typename BufferType>
320size_t BinauralSurroundRendererImpl::AddInputBufferTemplated(
321 const BufferType input_buffer_ptr, size_t num_channels, size_t num_frames) {
322 DCHECK_NE(surround_format_, kInvalid);
323 if (num_channels != num_input_channels_) {
324 LOG(WARNING) << "Invalid number of input channels";
325 return 0;
326 }
327
328 if (num_zero_padded_frames_ > 0) {
329 LOG(WARNING) << "Zero padded output buffers must be consumed prior to "
330 "|AddInterleavedBuffer| calls";
331 return 0;
332 }
333 const size_t num_available_input_frames =
334 std::min(a: num_frames, b: GetNumAvailableFramesInInputBuffer());
335
336 buffer_partitioner_->AddBuffer(input_buffer_ptr, num_input_channels_,
337 num_available_input_frames);
338 total_frames_buffered_ += num_available_input_frames;
339 return num_available_input_frames;
340}
341
342size_t BinauralSurroundRendererImpl::GetAvailableFramesInStereoOutputBuffer()
343 const {
344 const size_t num_available_samples_in_buffers =
345 (input_audio_buffer_queue_->Size() * frames_per_buffer_) +
346 buffer_unpartitioner_->GetNumBufferedFrames();
347 return std::min(a: total_frames_buffered_, b: num_available_samples_in_buffers);
348}
349
350size_t BinauralSurroundRendererImpl::GetInterleavedStereoOutput(
351 int16* output_buffer_ptr, size_t num_frames) {
352 return GetStereoOutputBufferTemplated<int16*>(output_buffer_ptr, num_frames);
353}
354
355size_t BinauralSurroundRendererImpl::GetInterleavedStereoOutput(
356 float* output_buffer_ptr, size_t num_frames) {
357 return GetStereoOutputBufferTemplated<float*>(output_buffer_ptr, num_frames);
358}
359
360size_t BinauralSurroundRendererImpl::GetPlanarStereoOutput(
361 int16** output_buffer_ptrs, size_t num_frames) {
362 return GetStereoOutputBufferTemplated<int16**>(output_buffer_ptr: output_buffer_ptrs,
363 num_frames);
364}
365
366size_t BinauralSurroundRendererImpl::GetPlanarStereoOutput(
367 float** output_buffer_ptrs, size_t num_frames) {
368 return GetStereoOutputBufferTemplated<float**>(output_buffer_ptr: output_buffer_ptrs,
369 num_frames);
370}
371
372template <typename BufferType>
373size_t BinauralSurroundRendererImpl::GetStereoOutputBufferTemplated(
374 BufferType output_buffer_ptr, size_t num_frames) {
375 DCHECK_NE(surround_format_, kInvalid);
376 const size_t num_frames_available = GetAvailableFramesInStereoOutputBuffer();
377 size_t num_frames_to_be_processed =
378 std::min(a: num_frames_available, b: num_frames);
379 if (num_frames_to_be_processed > total_frames_buffered_) {
380 // Avoid outputting zero padded input frames from |TriggerProcessing|
381 // calls.
382 num_frames_to_be_processed = total_frames_buffered_;
383 }
384
385 const size_t num_frames_written = buffer_unpartitioner_->GetBuffer(
386 output_buffer_ptr, kNumStereoChannels, num_frames_to_be_processed);
387
388 DCHECK_GE(total_frames_buffered_, num_frames_written);
389 total_frames_buffered_ -= num_frames_written;
390
391 if (total_frames_buffered_ == 0) {
392 // Clear zero padded frames from |TriggerProcessing| calls.
393 buffer_unpartitioner_->Clear();
394 num_zero_padded_frames_ = 0;
395 }
396
397 return num_frames_written;
398}
399
400void BinauralSurroundRendererImpl::Clear() {
401 input_audio_buffer_queue_->Clear();
402 buffer_partitioner_->Clear();
403 buffer_unpartitioner_->Clear();
404 total_frames_buffered_ = 0;
405 num_zero_padded_frames_ = 0;
406}
407
408bool BinauralSurroundRendererImpl::TriggerProcessing() {
409 if (num_zero_padded_frames_ > 0) {
410 LOG(WARNING) << "Zero padded output buffers must be consumed prior to "
411 "|TriggerProcessing| calls";
412 return false;
413 }
414 num_zero_padded_frames_ = buffer_partitioner_->Flush();
415 return num_zero_padded_frames_ > 0;
416}
417
418void BinauralSurroundRendererImpl::SetHeadRotation(float w, float x, float y,
419 float z) {
420 resonance_audio_api_->SetHeadRotation(x, y, z, w);
421}
422
423AudioBuffer* BinauralSurroundRendererImpl::ProcessBuffer() {
424 if (input_audio_buffer_queue_->Size() == 0) {
425 LOG(WARNING) << "Buffer underflow detected";
426 return nullptr;
427 }
428
429 const AudioBuffer* input = input_audio_buffer_queue_->AcquireOutputObject();
430 DCHECK_EQ(input->num_frames(), frames_per_buffer_);
431 DCHECK_EQ(num_input_channels_, input->num_channels());
432 GetRawChannelDataPointersFromAudioBuffer(audio_buffer: *input, channel_ptr_vector: &temp_planar_buffer_ptrs_);
433 // Initialize surround rendering.
434 const float* planar_ptr;
435
436 switch (surround_format_) {
437 case kSurroundMono:
438 case kSurroundStereo:
439 case kSurroundFiveDotOne:
440 case kSurroundSevenDotOne:
441 DCHECK_EQ(input->num_channels(), source_ids_.size());
442 for (size_t source_itr = 0; source_itr < source_ids_.size();
443 ++source_itr) {
444 planar_ptr = (*input)[source_itr].begin();
445 resonance_audio_api_->SetPlanarBuffer(source_id: source_ids_[source_itr],
446 audio_buffer_ptr: &planar_ptr, num_channels: kNumMonoChannels,
447 num_frames: input->num_frames());
448 }
449 break;
450 case kFirstOrderAmbisonics:
451 case kSecondOrderAmbisonics:
452 case kThirdOrderAmbisonics:
453 DCHECK_EQ(source_ids_.size(), 1U);
454 resonance_audio_api_->SetPlanarBuffer(
455 source_id: source_ids_[0], audio_buffer_ptr: temp_planar_buffer_ptrs_.data(),
456 num_channels: input->num_channels(), num_frames: input->num_frames());
457 break;
458 case kFirstOrderAmbisonicsWithNonDiegeticStereo:
459 case kSecondOrderAmbisonicsWithNonDiegeticStereo:
460 case kThirdOrderAmbisonicsWithNonDiegeticStereo:
461 DCHECK_EQ(source_ids_.size(), 2U);
462 DCHECK_GT(input->num_channels(), kNumStereoChannels);
463 static_cast<ResonanceAudioApiImpl*>(resonance_audio_api_.get())
464 ->SetPlanarBuffer(source_id: source_ids_[0], audio_buffer_ptr: temp_planar_buffer_ptrs_.data(),
465 num_channels: input->num_channels() - kNumStereoChannels,
466 num_frames: input->num_frames());
467 static_cast<ResonanceAudioApiImpl*>(resonance_audio_api_.get())
468 ->SetPlanarBuffer(source_id: source_ids_[1],
469 audio_buffer_ptr: temp_planar_buffer_ptrs_.data() +
470 (input->num_channels() - kNumStereoChannels),
471 num_channels: kNumStereoChannels, num_frames: input->num_frames());
472 break;
473 default:
474 LOG(FATAL) << "Undefined surround format";
475 break;
476 }
477
478 // Create a copy of the processed |AudioBuffer| to pass it to output buffer
479 // queue.
480 auto* const vraudio_api_impl =
481 static_cast<ResonanceAudioApiImpl*>(resonance_audio_api_.get());
482 vraudio_api_impl->ProcessNextBuffer();
483 output_buffer_ = *vraudio_api_impl->GetStereoOutputBuffer();
484
485 if (output_gain_ != 1.0f) {
486 for (AudioBuffer::Channel& channel : output_buffer_) {
487 ScalarMultiply(length: output_buffer_.num_frames(), gain: output_gain_, input: channel.begin(),
488 output: channel.begin());
489 }
490 }
491 input_audio_buffer_queue_->ReleaseOutputObject(object: input);
492 return &output_buffer_;
493}
494
495} // namespace vraudio
496

source code of qtmultimedia/src/3rdparty/resonance-audio/resonance_audio/graph/binaural_surround_renderer_impl.cc