1/*
2Copyright 2018 Google Inc. All Rights Reserved.
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS-IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/
16
17#include "graph/resonance_audio_api_impl.h"
18
19#include <algorithm>
20#include <numeric>
21
22#include "ambisonics/utils.h"
23#include "base/constants_and_types.h"
24#include "base/logging.h"
25#include "base/misc_math.h"
26#include "base/source_parameters.h"
27
28#include "base/unique_ptr_wrapper.h"
29#include "config/source_config.h"
30#include "dsp/channel_converter.h"
31#include "dsp/distance_attenuation.h"
32#include "graph/source_parameters_manager.h"
33#include "utils/planar_interleaved_conversion.h"
34#include "utils/sample_type_conversion.h"
35
36namespace vraudio {
37
38namespace {
39
40// Support 50 setter calls for 512 sources.
41const size_t kMaxNumTasksOnTaskQueue = 50 * 512;
42
43// User warning/notification messages.
44static const char* kBadInputPointerMessage = "Ignoring nullptr buffer";
45static const char* kBufferSizeMustMatchNumFramesMessage =
46 "Number of frames must match the frames per buffer specified during "
47 "construction - ignoring buffer";
48
49// Helper method to fetch |SourceGraphConfig| from |RenderingMode|.
50SourceGraphConfig GetSourceGraphConfigFromRenderingMode(
51 RenderingMode rendering_mode) {
52 switch (rendering_mode) {
53 case RenderingMode::kStereoPanning:
54 return StereoPanningConfig();
55 case RenderingMode::kBinauralLowQuality:
56 return BinauralLowQualityConfig();
57 case RenderingMode::kBinauralMediumQuality:
58 return BinauralMediumQualityConfig();
59 case RenderingMode::kBinauralHighQuality:
60 return BinauralHighQualityConfig();
61 case RenderingMode::kRoomEffectsOnly:
62 return RoomEffectsOnlyConfig();
63 default:
64 LOG(FATAL) << "Unknown rendering mode";
65 break;
66 }
67 return BinauralHighQualityConfig();
68}
69
70} // namespace
71
72ResonanceAudioApiImpl::ResonanceAudioApiImpl(size_t num_channels,
73 size_t frames_per_buffer,
74 int sample_rate_hz)
75 : system_settings_(num_channels, frames_per_buffer, sample_rate_hz),
76 task_queue_(kMaxNumTasksOnTaskQueue),
77 source_id_counter_(0) {
78 if (num_channels != kNumStereoChannels) {
79 LOG(FATAL) << "Only stereo output is supported";
80 return;
81 }
82
83 if (frames_per_buffer > kMaxSupportedNumFrames) {
84 LOG(FATAL) << "Only frame lengths up to " << kMaxSupportedNumFrames
85 << " are supported.";
86 return;
87 }
88
89 // The pffft library requires a minimum buffer size of 32 samples.
90 if (frames_per_buffer < FftManager::kMinFftSize) {
91 LOG(FATAL) << "The minimum number of frames per buffer is "
92 << FftManager::kMinFftSize << " samples";
93 return;
94 }
95 graph_manager_.reset(p: new GraphManager(system_settings_));
96}
97
98ResonanceAudioApiImpl::~ResonanceAudioApiImpl() {
99 // Clear task queue before shutting down.
100 task_queue_.Execute();
101}
102
103bool ResonanceAudioApiImpl::FillInterleavedOutputBuffer(size_t num_channels,
104 size_t num_frames,
105 float* buffer_ptr) {
106 DCHECK(buffer_ptr);
107 return FillOutputBuffer<float*>(num_channels, num_frames, buffer_ptr);
108}
109
110bool ResonanceAudioApiImpl::FillInterleavedOutputBuffer(size_t num_channels,
111 size_t num_frames,
112 int16* buffer_ptr) {
113 DCHECK(buffer_ptr);
114 return FillOutputBuffer<int16*>(num_channels, num_frames, buffer_ptr);
115}
116
117bool ResonanceAudioApiImpl::FillPlanarOutputBuffer(size_t num_channels,
118 size_t num_frames,
119 float* const* buffer_ptr) {
120 DCHECK(buffer_ptr);
121 return FillOutputBuffer<float* const*>(num_channels, num_frames, buffer_ptr);
122}
123
124bool ResonanceAudioApiImpl::FillPlanarOutputBuffer(size_t num_channels,
125 size_t num_frames,
126 int16* const* buffer_ptr) {
127 DCHECK(buffer_ptr);
128 return FillOutputBuffer<int16* const*>(num_channels, num_frames, buffer_ptr);
129}
130
131void ResonanceAudioApiImpl::SetHeadPosition(float x, float y, float z) {
132 auto task = [this, x, y, z]() {
133 const WorldPosition head_position(x, y, z);
134 system_settings_.SetHeadPosition(head_position);
135 };
136 task_queue_.Post(task);
137}
138
139void ResonanceAudioApiImpl::SetHeadRotation(float x, float y, float z,
140 float w) {
141 auto task = [this, w, x, y, z]() {
142 const WorldRotation head_rotation(w, x, y, z);
143 system_settings_.SetHeadRotation(head_rotation);
144 };
145 task_queue_.Post(task);
146}
147
148void ResonanceAudioApiImpl::SetMasterVolume(float volume) {
149 auto task = [this, volume]() { system_settings_.SetMasterGain(volume); };
150 task_queue_.Post(task);
151}
152
153int ResonanceAudioApiImpl::CreateAmbisonicSource(size_t num_channels) {
154 if (num_channels < kNumFirstOrderAmbisonicChannels ||
155 !IsValidAmbisonicOrder(num_channels)) {
156 // Invalid number of input channels, don't create the ambisonic source.
157 LOG(ERROR) << "Invalid number of channels for the ambisonic source: "
158 << num_channels;
159 return kInvalidSourceId;
160 }
161
162 const int ambisonic_source_id = source_id_counter_.fetch_add(i: 1);
163
164 const size_t num_valid_channels =
165 std::min(a: num_channels, b: graph_manager_->GetNumMaxAmbisonicChannels());
166 if (num_valid_channels < num_channels) {
167 LOG(WARNING) << "Number of ambisonic channels will be diminished to "
168 << num_valid_channels;
169 }
170
171 auto task = [this, ambisonic_source_id, num_valid_channels]() {
172 graph_manager_->CreateAmbisonicSource(ambisonic_source_id,
173 num_channels: num_valid_channels);
174 system_settings_.GetSourceParametersManager()->Register(
175 source_id: ambisonic_source_id);
176 // Overwrite default source parameters for ambisonic source.
177 auto source_parameters =
178 system_settings_.GetSourceParametersManager()->GetMutableParameters(
179 source_id: ambisonic_source_id);
180 source_parameters->room_effects_gain = 0.0f;
181 source_parameters->distance_rolloff_model = DistanceRolloffModel::kNone;
182 source_parameters->distance_attenuation = 1.0f;
183 };
184 task_queue_.Post(task);
185 return ambisonic_source_id;
186}
187
188int ResonanceAudioApiImpl::CreateStereoSource(size_t num_channels) {
189 if (num_channels > kNumStereoChannels) {
190 LOG(ERROR) << "Unsupported number of input channels";
191 return kInvalidSourceId;
192 }
193 const int stereo_source_id = source_id_counter_.fetch_add(i: 1);
194
195 auto task = [this, stereo_source_id]() {
196 graph_manager_->CreateStereoSource(stereo_source_id);
197 system_settings_.GetSourceParametersManager()->Register(source_id: stereo_source_id);
198 auto source_parameters =
199 system_settings_.GetSourceParametersManager()->GetMutableParameters(
200 source_id: stereo_source_id);
201 source_parameters->enable_hrtf = false;
202 };
203 task_queue_.Post(task);
204 return stereo_source_id;
205}
206
207int ResonanceAudioApiImpl::CreateSoundObjectSource(
208 RenderingMode rendering_mode) {
209 const int sound_object_source_id = source_id_counter_.fetch_add(i: 1);
210
211 const auto config = GetSourceGraphConfigFromRenderingMode(rendering_mode);
212 auto task = [this, sound_object_source_id, config]() {
213 graph_manager_->CreateSoundObjectSource(
214 sound_object_source_id, ambisonic_order: config.ambisonic_order, enable_hrtf: config.enable_hrtf,
215 enable_direct_rendering: config.enable_direct_rendering);
216 system_settings_.GetSourceParametersManager()->Register(
217 source_id: sound_object_source_id);
218 auto source_parameters =
219 system_settings_.GetSourceParametersManager()->GetMutableParameters(
220 source_id: sound_object_source_id);
221 source_parameters->enable_hrtf = config.enable_hrtf;
222 };
223 task_queue_.Post(task);
224 return sound_object_source_id;
225}
226
227void ResonanceAudioApiImpl::DestroySource(SourceId source_id) {
228 auto task = [this, source_id]() {
229 graph_manager_->DestroySource(source_id);
230 system_settings_.GetSourceParametersManager()->Unregister(source_id);
231 };
232 task_queue_.Post(task);
233}
234
235void ResonanceAudioApiImpl::SetInterleavedBuffer(SourceId source_id,
236 const float* audio_buffer_ptr,
237 size_t num_channels,
238 size_t num_frames) {
239 SetSourceBuffer<const float*>(source_id, audio_buffer_ptr, num_input_channels: num_channels,
240 num_frames);
241}
242
243void ResonanceAudioApiImpl::SetInterleavedBuffer(SourceId source_id,
244 const int16* audio_buffer_ptr,
245 size_t num_channels,
246 size_t num_frames) {
247 SetSourceBuffer<const int16*>(source_id, audio_buffer_ptr, num_input_channels: num_channels,
248 num_frames);
249}
250
251void ResonanceAudioApiImpl::SetPlanarBuffer(
252 SourceId source_id, const float* const* audio_buffer_ptr,
253 size_t num_channels, size_t num_frames) {
254 SetSourceBuffer<const float* const*>(source_id, audio_buffer_ptr,
255 num_input_channels: num_channels, num_frames);
256}
257
258void ResonanceAudioApiImpl::SetPlanarBuffer(
259 SourceId source_id, const int16* const* audio_buffer_ptr,
260 size_t num_channels, size_t num_frames) {
261 SetSourceBuffer<const int16* const*>(source_id, audio_buffer_ptr,
262 num_input_channels: num_channels, num_frames);
263}
264
265void ResonanceAudioApiImpl::SetSourceDistanceAttenuation(
266 SourceId source_id, float distance_attenuation) {
267 auto task = [this, source_id, distance_attenuation]() {
268 auto source_parameters =
269 system_settings_.GetSourceParametersManager()->GetMutableParameters(
270 source_id);
271 if (source_parameters != nullptr) {
272 const auto& rolloff_model = source_parameters->distance_rolloff_model;
273 DCHECK_EQ(rolloff_model, DistanceRolloffModel::kNone);
274 if (rolloff_model != DistanceRolloffModel::kNone) {
275 LOG(WARNING) << "Implicit distance rolloff model is set. The value "
276 "will be overwritten.";
277 }
278 source_parameters->distance_attenuation = distance_attenuation;
279 }
280 };
281 task_queue_.Post(task);
282}
283
284void ResonanceAudioApiImpl::SetSourceDistanceModel(SourceId source_id,
285 DistanceRolloffModel rolloff,
286 float min_distance,
287 float max_distance) {
288 if (max_distance < min_distance && rolloff != DistanceRolloffModel::kNone) {
289 LOG(WARNING) << "max_distance must be larger than min_distance";
290 return;
291 }
292 auto task = [this, source_id, rolloff, min_distance, max_distance]() {
293 auto source_parameters =
294 system_settings_.GetSourceParametersManager()->GetMutableParameters(
295 source_id);
296 if (source_parameters != nullptr) {
297 source_parameters->distance_rolloff_model = rolloff;
298 source_parameters->minimum_distance = min_distance;
299 source_parameters->maximum_distance = max_distance;
300 }
301 };
302 task_queue_.Post(task);
303}
304
305void ResonanceAudioApiImpl::SetSourcePosition(SourceId source_id, float x,
306 float y, float z) {
307 const WorldPosition position(x, y, z);
308 auto task = [this, source_id, position]() {
309 auto source_parameters =
310 system_settings_.GetSourceParametersManager()->GetMutableParameters(
311 source_id);
312 if (source_parameters != nullptr) {
313 source_parameters->object_transform.position = position;
314 }
315 };
316 task_queue_.Post(task);
317}
318
319void ResonanceAudioApiImpl::SetSourceRoomEffectsGain(SourceId source_id,
320 float room_effects_gain) {
321 auto task = [this, source_id, room_effects_gain]() {
322 auto source_parameters =
323 system_settings_.GetSourceParametersManager()->GetMutableParameters(
324 source_id);
325 if (source_parameters != nullptr) {
326 source_parameters->room_effects_gain = room_effects_gain;
327 }
328 };
329 task_queue_.Post(task);
330}
331
332void ResonanceAudioApiImpl::SetSourceRotation(SourceId source_id, float x,
333 float y, float z, float w) {
334 const WorldRotation rotation(w, x, y, z);
335 auto task = [this, source_id, rotation]() {
336 auto source_parameters =
337 system_settings_.GetSourceParametersManager()->GetMutableParameters(
338 source_id);
339 if (source_parameters != nullptr) {
340 source_parameters->object_transform.rotation = rotation;
341 }
342 };
343 task_queue_.Post(task);
344}
345
346void ResonanceAudioApiImpl::SetSourceVolume(SourceId source_id, float volume) {
347 auto task = [this, source_id, volume]() {
348 auto source_parameters =
349 system_settings_.GetSourceParametersManager()->GetMutableParameters(
350 source_id);
351 if (source_parameters != nullptr) {
352 source_parameters->gain = volume;
353 }
354 };
355 task_queue_.Post(task);
356}
357
358void ResonanceAudioApiImpl::SetSoundObjectDirectivity(
359 SourceId sound_object_source_id, float alpha, float order) {
360 auto task = [this, sound_object_source_id, alpha, order]() {
361 auto source_parameters =
362 system_settings_.GetSourceParametersManager()->GetMutableParameters(
363 source_id: sound_object_source_id);
364 if (source_parameters != nullptr) {
365 source_parameters->directivity_alpha = alpha;
366 source_parameters->directivity_order = order;
367 }
368 };
369 task_queue_.Post(task);
370}
371
372void ResonanceAudioApiImpl::SetSoundObjectListenerDirectivity(
373 SourceId sound_object_source_id, float alpha, float order) {
374 auto task = [this, sound_object_source_id, alpha, order]() {
375 auto source_parameters =
376 system_settings_.GetSourceParametersManager()->GetMutableParameters(
377 source_id: sound_object_source_id);
378 if (source_parameters != nullptr) {
379 source_parameters->listener_directivity_alpha = alpha;
380 source_parameters->listener_directivity_order = order;
381 }
382 };
383 task_queue_.Post(task);
384}
385
386void ResonanceAudioApiImpl::SetSoundObjectNearFieldEffectGain(
387 SourceId sound_object_source_id, float gain) {
388 auto task = [this, sound_object_source_id, gain]() {
389 auto source_parameters =
390 system_settings_.GetSourceParametersManager()->GetMutableParameters(
391 source_id: sound_object_source_id);
392 if (source_parameters != nullptr) {
393 source_parameters->near_field_gain = gain;
394 }
395 };
396 task_queue_.Post(task);
397}
398
399void ResonanceAudioApiImpl::SetSoundObjectOcclusionIntensity(
400 SourceId sound_object_source_id, float intensity) {
401 auto task = [this, sound_object_source_id, intensity]() {
402 auto source_parameters =
403 system_settings_.GetSourceParametersManager()->GetMutableParameters(
404 source_id: sound_object_source_id);
405 if (source_parameters != nullptr) {
406 source_parameters->occlusion_intensity = intensity;
407 }
408 };
409 task_queue_.Post(task);
410}
411
412void ResonanceAudioApiImpl::SetSoundObjectSpread(
413 SourceId sound_object_source_id, float spread_deg) {
414 auto task = [this, sound_object_source_id, spread_deg]() {
415 auto source_parameters =
416 system_settings_.GetSourceParametersManager()->GetMutableParameters(
417 source_id: sound_object_source_id);
418 if (source_parameters != nullptr) {
419 source_parameters->spread_deg = spread_deg;
420 }
421 };
422 task_queue_.Post(task);
423}
424
425void ResonanceAudioApiImpl::EnableRoomEffects(bool enable) {
426 auto task = [this, enable]() { graph_manager_->EnableRoomEffects(enable); };
427 task_queue_.Post(task);
428}
429
430void ResonanceAudioApiImpl::SetReflectionProperties(
431 const ReflectionProperties& reflection_properties) {
432 auto task = [this, reflection_properties]() {
433 system_settings_.SetReflectionProperties(reflection_properties);
434 };
435 task_queue_.Post(task);
436}
437
438void ResonanceAudioApiImpl::SetReverbProperties(
439 const ReverbProperties& reverb_properties) {
440 auto task = [this, reverb_properties]() {
441 system_settings_.SetReverbProperties(reverb_properties);
442 };
443 task_queue_.Post(task);
444}
445
446const AudioBuffer* ResonanceAudioApiImpl::GetAmbisonicOutputBuffer() const {
447 return graph_manager_->GetAmbisonicBuffer();
448}
449
450const AudioBuffer* ResonanceAudioApiImpl::GetStereoOutputBuffer() const {
451 return graph_manager_->GetStereoBuffer();
452}
453
454const AudioBuffer *ResonanceAudioApiImpl::GetReverbBuffer() const {
455 return graph_manager_->GetReverbBuffer();
456}
457
458void ResonanceAudioApiImpl::ProcessNextBuffer() {
459#if defined(ENABLE_TRACING) && !ION_PRODUCTION
460 // This enables tracing on the audio thread.
461 auto task = []() { ENABLE_TRACING_ON_CURRENT_THREAD("AudioThread"); };
462 task_queue_.Post(task);
463#endif // defined(ENABLE_TRACING) && !ION_PRODUCTION
464
465
466 task_queue_.Execute();
467
468 // Update room effects only if the pipeline is initialized.
469 if (graph_manager_->GetRoomEffectsEnabled()) {
470 graph_manager_->UpdateRoomReflections();
471 graph_manager_->UpdateRoomReverb();
472 }
473 // Update source attenuation parameters.
474 const auto process = [this](SourceParameters* parameters) {
475 const float master_gain = system_settings_.GetMasterGain();
476 const auto& listener_position = system_settings_.GetHeadPosition();
477 const auto& reflection_properties =
478 system_settings_.GetReflectionProperties();
479 const auto& reverb_properties = system_settings_.GetReverbProperties();
480 UpdateAttenuationParameters(master_gain, reflections_gain: reflection_properties.gain,
481 reverb_gain: reverb_properties.gain, listener_position,
482 parameters);
483 };
484 system_settings_.GetSourceParametersManager()->ProcessAllParameters(process);
485
486 graph_manager_->Process();
487}
488
489void ResonanceAudioApiImpl::SetStereoSpeakerMode(bool enabled) {
490 auto task = [this, enabled]() {
491 system_settings_.SetStereoSpeakerMode(enabled);
492 };
493 task_queue_.Post(task);
494}
495
496template <typename OutputType>
497bool ResonanceAudioApiImpl::FillOutputBuffer(size_t num_channels,
498 size_t num_frames,
499 OutputType buffer_ptr) {
500
501
502 if (buffer_ptr == nullptr) {
503 LOG(WARNING) << kBadInputPointerMessage;
504 return false;
505 }
506 if (num_channels != kNumStereoChannels) {
507 LOG(WARNING) << "Output buffer must be stereo";
508 return false;
509 }
510 const size_t num_input_samples = num_frames * num_channels;
511 const size_t num_expected_output_samples =
512 system_settings_.GetFramesPerBuffer() * system_settings_.GetNumChannels();
513 if (num_input_samples != num_expected_output_samples) {
514 LOG(WARNING) << "Output buffer size must be " << num_expected_output_samples
515 << " samples";
516 return false;
517 }
518
519 // Get the processed output buffer.
520 ProcessNextBuffer();
521 const AudioBuffer* output_buffer = GetStereoOutputBuffer();
522 if (output_buffer == nullptr) {
523 // This indicates that the graph processing is triggered without having any
524 // connected sources.
525 return false;
526 }
527
528 FillExternalBuffer(*output_buffer, buffer_ptr, num_frames, num_channels);
529 return true;
530}
531
532template <typename SampleType>
533void ResonanceAudioApiImpl::SetSourceBuffer(SourceId source_id,
534 SampleType audio_buffer_ptr,
535 size_t num_input_channels,
536 size_t num_frames) {
537 // Execute task queue to ensure newly created sound sources are initialized.
538 task_queue_.Execute();
539
540 if (audio_buffer_ptr == nullptr) {
541 LOG(WARNING) << kBadInputPointerMessage;
542 return;
543 }
544 if (num_frames != system_settings_.GetFramesPerBuffer()) {
545 LOG(WARNING) << kBufferSizeMustMatchNumFramesMessage;
546 return;
547 }
548
549 AudioBuffer* const output_buffer =
550 graph_manager_->GetMutableAudioBuffer(source_id);
551 if (output_buffer == nullptr) {
552 LOG(WARNING) << "Source audio buffer not found";
553 return;
554 }
555 const size_t num_output_channels = output_buffer->num_channels();
556
557 if (num_input_channels == num_output_channels) {
558 FillAudioBuffer(audio_buffer_ptr, num_frames, num_input_channels,
559 output_buffer);
560
561 return;
562 }
563
564 if ((num_input_channels == kNumMonoChannels) &&
565 (num_output_channels == kNumStereoChannels)) {
566 FillAudioBufferWithChannelRemapping(
567 audio_buffer_ptr, num_frames, num_input_channels,
568 {0, 0} /* channel_map */, output_buffer);
569 return;
570 }
571
572 if (num_input_channels > num_output_channels) {
573 std::vector<size_t> channel_map(num_output_channels);
574 // Fill channel map with increasing indices.
575 std::iota(first: std::begin(cont&: channel_map), last: std::end(cont&: channel_map), value: 0);
576 FillAudioBufferWithChannelRemapping(audio_buffer_ptr, num_frames,
577 num_input_channels, channel_map,
578 output_buffer);
579 return;
580 }
581
582 LOG(WARNING) << "Number of input channels does not match the number of "
583 "output channels";
584}
585
586} // namespace vraudio
587

source code of qtmultimedia/src/3rdparty/resonance-audio/resonance_audio/graph/resonance_audio_api_impl.cc