Skip to content

Commit 345f8b7

Browse files
hiroshihoriehiroshihorie
committed
Manual rendering
1 parent be003d5 commit 345f8b7

File tree

2 files changed

+208
-6
lines changed

2 files changed

+208
-6
lines changed

‎modules/audio_device/audio_engine_device.h‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,7 @@ class AudioEngineDevice : public AudioDeviceModule,
264264
bool DidEnableInput() const {
265265
return !prev.IsInputEnabled() && next.IsInputEnabled();
266266
}
267+
267268
bool DidEnableOutputOrInput() const {
268269
return DidEnableOutput() || DidEnableInput();
269270
}
@@ -301,6 +302,16 @@ class AudioEngineDevice : public AudioDeviceModule,
301302
return (prev.IsOutputEnabled() && next.IsOutputEnabled()) &&
302303
(prev.IsInputEnabled() && !next.IsInputEnabled());
303304
}
305+
306+
bool DidEnableManualRenderingMode() const {
307+
return prev.render_mode != RenderMode::Manual &&
308+
next.render_mode == RenderMode::Manual;
309+
}
310+
311+
bool DidEnableDeviceRenderingMode() const {
312+
return prev.render_mode != RenderMode::Device &&
313+
next.render_mode == RenderMode::Device;
314+
}
304315
};
305316

306317
EngineState engine_state_ RTC_GUARDED_BY(thread_);
@@ -310,7 +321,8 @@ class AudioEngineDevice : public AudioDeviceModule,
310321

311322
bool IsMicrophonePermissionGranted();
312323
void SetEngineState(std::function<EngineState(EngineState)> state_transform);
313-
void UpdateEngineState(EngineStateUpdate state);
324+
void UpdateDeviceEngineState(EngineStateUpdate state);
325+
void UpdateManualEngineState(EngineStateUpdate state);
314326

315327
// AudioEngine observer methods. May be called from any thread.
316328
void OnEngineConfigurationChange();
@@ -356,6 +368,7 @@ class AudioEngineDevice : public AudioDeviceModule,
356368

357369
// AVAudioEngine objects
358370
AVAudioEngine* engine_device_;
371+
AVAudioEngine* engine_manual_input_;
359372

360373
// Used for manual rendering mode
361374
AVAudioFormat* manual_render_rtc_format_; // Int16

‎modules/audio_device/audio_engine_device.mm‎

Lines changed: 194 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@
5252
const UInt16 kStartEngineMaxRetries = 10; // Maximum blocking 1sec.
5353
const useconds_t kStartEngineRetryDelayMs = 100;
5454

55-
// const size_t kMaximumFramesPerBuffer = 3072; // Maximum slice size for VoiceProcessingIO
55+
const size_t kMaximumFramesPerBuffer = 3072;
5656
const size_t kAudioSampleSize = 2; // Signed 16-bit integer
5757

5858
AudioEngineDevice::AudioEngineDevice(bool bypass_voice_processing)
@@ -857,9 +857,29 @@
857857
RTC_DCHECK(new_state.IsOutputEnabled());
858858
}
859859

860-
// Apply engine state changes
860+
// Save new state
861861
engine_state_ = new_state;
862-
UpdateEngineState(state);
862+
863+
// Did switch Device -> Manual rendering
864+
if (state.DidEnableManualRenderingMode()) {
865+
EngineStateUpdate shutdown_state = state; // Copy current state
866+
shutdown_state.next = {}; // Reset next state to default
867+
UpdateDeviceEngineState(shutdown_state); // Shutdown device rendering
868+
EngineStateUpdate startup_state = state; // Copy current state
869+
shutdown_state.prev = {}; //
870+
UpdateManualEngineState(startup_state); // Start manual mode
871+
} else if (state.DidEnableDeviceRenderingMode()) {
872+
EngineStateUpdate shutdown_state = state;
873+
shutdown_state.next = {}; // Reset next state to default
874+
UpdateManualEngineState(shutdown_state); // Shutdown manual rendering
875+
EngineStateUpdate startup_state = state; // Copy current state
876+
shutdown_state.prev = {}; //
877+
UpdateDeviceEngineState(startup_state); // Start device mode
878+
} else if (new_state.render_mode == RenderMode::Device) {
879+
UpdateDeviceEngineState(state);
880+
} else if (new_state.render_mode == RenderMode::Manual) {
881+
UpdateManualEngineState(state);
882+
}
863883

864884
// Buffer should be playing if output is running.
865885
if (new_state.IsOutputEnabled()) {
@@ -876,8 +896,177 @@
876896
}
877897
}
878898

879-
void AudioEngineDevice::UpdateEngineState(EngineStateUpdate state) {
899+
void AudioEngineDevice::UpdateManualEngineState(EngineStateUpdate state) {
900+
RTC_DCHECK_RUN_ON(thread_);
901+
RTC_DCHECK(engine_device_ == nullptr);
902+
903+
if (state.prev.IsAnyRunning() && !state.next.IsAnyRunning()) {
904+
LOGI() << "Stopping AVAudioEngine...";
905+
RTC_DCHECK(engine_manual_input_ != nil);
906+
[engine_manual_input_ stop];
907+
908+
LOGI() << "Stopping render thread...";
909+
RTC_DCHECK(render_thread_ != nullptr);
910+
render_thread_->Stop();
911+
render_thread_ = nullptr;
912+
913+
LOGI() << "Releasing render buffer...";
914+
RTC_DCHECK(render_buffer_ != nullptr);
915+
render_buffer_ = nullptr;
916+
917+
if (observer_ != nullptr) {
918+
observer_->OnEngineDidStop(engine_manual_input_, state.next.IsOutputEnabled(),
919+
state.next.IsInputEnabled());
920+
}
921+
}
922+
923+
if (state.next.IsAnyEnabled() && !state.prev.IsAnyEnabled()) {
924+
LOGI() << "Creating AVAudioEngine (manual)...";
925+
RTC_DCHECK(engine_manual_input_ == nullptr);
926+
engine_manual_input_ = [[AVAudioEngine alloc] init];
927+
928+
NSError* error = nil;
929+
BOOL result =
930+
[engine_manual_input_ enableManualRenderingMode:AVAudioEngineManualRenderingModeRealtime
931+
format:manual_render_rtc_format_
932+
maximumFrameCount:kMaximumFramesPerBuffer
933+
error:&error];
934+
if (!result) {
935+
LOGE() << "Failed to set manual rendering mode: " << error.localizedDescription.UTF8String;
936+
}
937+
938+
// Assign manual rendering block
939+
render_block_ = engine_manual_input_.manualRenderingBlock;
940+
941+
if (observer_ != nullptr) {
942+
observer_->OnEngineDidCreate(engine_manual_input_);
943+
}
944+
}
945+
946+
if (!state.next.IsOutputEnabled() && audio_device_buffer_->IsPlaying()) {
947+
LOGI() << "Stopping Playout buffer...";
948+
if (engine_device_ != nullptr) {
949+
// Rendering must be stopped first.
950+
RTC_DCHECK(!engine_device_.running);
951+
}
952+
audio_device_buffer_->StopPlayout();
953+
}
954+
955+
if (!state.next.IsInputEnabled() && audio_device_buffer_->IsRecording()) {
956+
LOGI() << "Stopping Record buffer...";
957+
if (engine_device_ != nullptr) {
958+
// Rendering must be stopped first.
959+
RTC_DCHECK(!engine_device_.running);
960+
}
961+
audio_device_buffer_->StopRecording();
962+
}
963+
964+
if (state.DidAnyEnable() && observer_ != nullptr) {
965+
// Invoke here before configuring nodes. In iOS, session configuration is required before
966+
// enabling AGC, muted talker etc.
967+
observer_->OnEngineWillEnable(engine_manual_input_, state.next.IsOutputEnabled(),
968+
state.next.IsInputEnabled());
969+
}
970+
971+
if (state.next.IsOutputEnabled() && !state.prev.IsOutputEnabled()) {
972+
LOGI() << "Enabling output for AVAudioEngine...";
973+
974+
audio_device_buffer_->SetPlayoutSampleRate(manual_render_rtc_format_.sampleRate);
975+
audio_device_buffer_->SetPlayoutChannels(manual_render_rtc_format_.channelCount);
976+
RTC_DCHECK(audio_device_buffer_ != nullptr);
977+
fine_audio_buffer_.reset(new FineAudioBuffer(audio_device_buffer_.get()));
978+
979+
} else if (state.prev.IsOutputEnabled() && !state.next.IsOutputEnabled()) {
980+
LOGI() << "Disabling output for AVAudioEngine...";
981+
RTC_DCHECK(!engine_manual_input_.running);
982+
}
983+
984+
if (state.next.IsInputEnabled() &&
985+
(!state.prev.IsInputEnabled() || state.IsEngineRecreateRequired())) {
986+
LOGI() << "Enabling input for AVAudioEngine...";
987+
988+
audio_device_buffer_->SetPlayoutSampleRate(manual_render_rtc_format_.sampleRate);
989+
audio_device_buffer_->SetPlayoutChannels(manual_render_rtc_format_.channelCount);
990+
RTC_DCHECK(audio_device_buffer_ != nullptr);
991+
fine_audio_buffer_.reset(new FineAudioBuffer(audio_device_buffer_.get()));
992+
993+
RTC_DCHECK(!engine_manual_input_.running);
994+
} else if (state.prev.IsInputEnabled() && !state.next.IsInputEnabled()) {
995+
LOGI() << "Disabling input for AVAudioEngine...";
996+
RTC_DCHECK(!engine_manual_input_.running);
997+
}
998+
999+
if (state.DidAnyDisable() && observer_ != nullptr) {
1000+
observer_->OnEngineDidDisable(engine_manual_input_, state.next.IsOutputEnabled(),
1001+
state.next.IsInputEnabled());
1002+
}
1003+
1004+
// Start playout buffer if output is running
1005+
if (state.next.IsOutputEnabled() && !audio_device_buffer_->IsPlaying()) {
1006+
if (engine_device_ != nullptr) {
1007+
// Rendering must be stopped first.
1008+
RTC_DCHECK(!engine_device_.running);
1009+
}
1010+
LOGI() << "Starting Playout buffer...";
1011+
audio_device_buffer_->StartPlayout();
1012+
fine_audio_buffer_->ResetPlayout();
1013+
}
1014+
1015+
// Start recording buffer if input is running
1016+
if (state.next.IsInputEnabled() && !audio_device_buffer_->IsRecording()) {
1017+
if (engine_device_ != nullptr) {
1018+
// Rendering must be stopped first.
1019+
RTC_DCHECK(!engine_device_.running);
1020+
}
1021+
LOGI() << "Starting Record buffer...";
1022+
audio_device_buffer_->StartRecording();
1023+
fine_audio_buffer_->ResetRecord();
1024+
}
1025+
1026+
if (state.next.IsAnyRunning() && !state.prev.IsAnyRunning()) {
1027+
if (observer_ != nullptr) {
1028+
observer_->OnEngineWillStart(engine_manual_input_, state.next.IsOutputEnabled(),
1029+
state.next.IsInputEnabled());
1030+
}
1031+
1032+
LOGI() << "Allocating render buffer...";
1033+
RTC_DCHECK(render_buffer_ == nullptr);
1034+
render_buffer_ = [[AVAudioPCMBuffer alloc] initWithPCMFormat:manual_render_rtc_format_
1035+
frameCapacity:kMaximumFramesPerBuffer];
1036+
1037+
LOGI() << "Starting AVAudioEngine...";
1038+
NSError* error = nil;
1039+
1040+
BOOL start_result = [engine_manual_input_ startAndReturnError:&error];
1041+
if (!start_result) {
1042+
LOGE() << "Failed to start engine after " << kStartEngineMaxRetries << " attempts";
1043+
DebugAudioEngine();
1044+
}
1045+
1046+
// Create render thread
1047+
LOGI() << "Starting render thread...";
1048+
RTC_DCHECK(render_thread_ == nullptr);
1049+
render_thread_ = rtc::Thread::Create();
1050+
render_thread_->SetName("render_thread", nullptr);
1051+
render_thread_->Start();
1052+
render_thread_->PostTask([this] {
1053+
// RTC_DCHECK_RUN_ON(thread_);
1054+
this->StartRenderLoop();
1055+
});
1056+
}
1057+
1058+
if (state.prev.IsAnyEnabled() && !state.next.IsAnyEnabled()) {
1059+
if (observer_ != nullptr) {
1060+
observer_->OnEngineWillRelease(engine_manual_input_);
1061+
}
1062+
LOGI() << "Releasing AVAudioEngine...";
1063+
engine_manual_input_ = nil;
1064+
}
1065+
}
1066+
1067+
void AudioEngineDevice::UpdateDeviceEngineState(EngineStateUpdate state) {
8801068
RTC_DCHECK_RUN_ON(thread_);
1069+
RTC_DCHECK(engine_manual_input_ == nullptr);
8811070

8821071
if (state.prev.IsAnyRunning() &&
8831072
(!state.next.IsAnyRunning() || state.DidUpdateAudioGraph() || state.DidBeginInterruption() ||
@@ -902,7 +1091,7 @@
9021091

9031092
if (state.next.IsAnyEnabled() &&
9041093
(!state.prev.IsAnyEnabled() || state.IsEngineRecreateRequired())) {
905-
LOGI() << "Creating AVAudioEngine...";
1094+
LOGI() << "Creating AVAudioEngine (device)...";
9061095
RTC_DCHECK(engine_device_ == nullptr);
9071096
engine_device_ = [[AVAudioEngine alloc] init];
9081097

0 commit comments

Comments
 (0)