Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 // Copyright 2013 The Chromium Authors. All rights reserved. | 1 // Copyright 2013 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 #include "content/renderer/media/media_stream_audio_processor.h" | 5 #include "content/renderer/media/media_stream_audio_processor.h" |
| 6 | 6 |
| 7 #include <stddef.h> | 7 #include <stddef.h> |
| 8 #include <stdint.h> | 8 #include <stdint.h> |
| 9 #include <utility> | 9 #include <utility> |
| 10 | 10 |
| 11 #include "base/command_line.h" | 11 #include "base/command_line.h" |
| 12 #include "base/metrics/field_trial.h" | 12 #include "base/metrics/field_trial.h" |
| 13 #include "base/metrics/histogram_macros.h" | 13 #include "base/metrics/histogram_macros.h" |
| 14 #include "base/single_thread_task_runner.h" | 14 #include "base/single_thread_task_runner.h" |
| 15 #include "base/strings/string_number_conversions.h" | 15 #include "base/strings/string_number_conversions.h" |
| 16 #include "base/threading/thread_task_runner_handle.h" | |
| 16 #include "base/trace_event/trace_event.h" | 17 #include "base/trace_event/trace_event.h" |
| 17 #include "build/build_config.h" | 18 #include "build/build_config.h" |
| 18 #include "content/public/common/content_switches.h" | 19 #include "content/public/common/content_switches.h" |
| 19 #include "content/renderer/media/media_stream_audio_processor_options.h" | 20 #include "content/renderer/media/media_stream_audio_processor_options.h" |
| 20 #include "content/renderer/media/webrtc_audio_device_impl.h" | 21 #include "content/renderer/media/webrtc_audio_device_impl.h" |
| 21 #include "media/base/audio_converter.h" | 22 #include "media/base/audio_converter.h" |
| 22 #include "media/base/audio_fifo.h" | 23 #include "media/base/audio_fifo.h" |
| 23 #include "media/base/audio_parameters.h" | 24 #include "media/base/audio_parameters.h" |
| 24 #include "media/base/channel_layout.h" | 25 #include "media/base/channel_layout.h" |
| 25 #include "third_party/WebKit/public/platform/WebMediaConstraints.h" | 26 #include "third_party/WebKit/public/platform/WebMediaConstraints.h" |
| (...skipping 255 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 281 // to Consume(). Only used when the FIFO is disabled. | 282 // to Consume(). Only used when the FIFO is disabled. |
| 282 bool data_available_; | 283 bool data_available_; |
| 283 }; | 284 }; |
| 284 | 285 |
| 285 MediaStreamAudioProcessor::MediaStreamAudioProcessor( | 286 MediaStreamAudioProcessor::MediaStreamAudioProcessor( |
| 286 const blink::WebMediaConstraints& constraints, | 287 const blink::WebMediaConstraints& constraints, |
| 287 const MediaStreamDevice::AudioDeviceParameters& input_params, | 288 const MediaStreamDevice::AudioDeviceParameters& input_params, |
| 288 WebRtcPlayoutDataSource* playout_data_source) | 289 WebRtcPlayoutDataSource* playout_data_source) |
| 289 : render_delay_ms_(0), | 290 : render_delay_ms_(0), |
| 290 playout_data_source_(playout_data_source), | 291 playout_data_source_(playout_data_source), |
| 291 main_thread_message_loop_(base::MessageLoop::current()), | 292 main_thread_runner_(base::ThreadTaskRunnerHandle::Get()), |
| 292 audio_mirroring_(false), | 293 audio_mirroring_(false), |
| 293 typing_detected_(false), | 294 typing_detected_(false), |
| 294 stopped_(false) { | 295 stopped_(false) { |
| 296 DCHECK(main_thread_runner_); | |
| 295 capture_thread_checker_.DetachFromThread(); | 297 capture_thread_checker_.DetachFromThread(); |
| 296 render_thread_checker_.DetachFromThread(); | 298 render_thread_checker_.DetachFromThread(); |
| 297 InitializeAudioProcessingModule(constraints, input_params); | 299 InitializeAudioProcessingModule(constraints, input_params); |
| 298 | 300 |
| 299 aec_dump_message_filter_ = AecDumpMessageFilter::Get(); | 301 aec_dump_message_filter_ = AecDumpMessageFilter::Get(); |
| 300 // In unit tests not creating a message filter, |aec_dump_message_filter_| | 302 // In unit tests not creating a message filter, |aec_dump_message_filter_| |
| 301 // will be NULL. We can just ignore that. Other unit tests and browser tests | 303 // will be NULL. We can just ignore that. Other unit tests and browser tests |
| 302 // ensure that we do get the filter when we should. | 304 // ensure that we do get the filter when we should. |
| 303 if (aec_dump_message_filter_.get()) | 305 if (aec_dump_message_filter_.get()) |
| 304 aec_dump_message_filter_->AddDelegate(this); | 306 aec_dump_message_filter_->AddDelegate(this); |
| 305 | 307 |
| 306 // Create and configure |audio_repetition_detector_|. | 308 // Create and configure |audio_repetition_detector_|. |
| 307 std::vector<int> look_back_times; | 309 std::vector<int> look_back_times; |
| 308 for (int time = kMaxLookbackTimeMs; time >= kMinLookbackTimeMs; | 310 for (int time = kMaxLookbackTimeMs; time >= kMinLookbackTimeMs; |
| 309 time -= kLookbackTimeStepMs) { | 311 time -= kLookbackTimeStepMs) { |
| 310 look_back_times.push_back(time); | 312 look_back_times.push_back(time); |
| 311 } | 313 } |
| 312 audio_repetition_detector_.reset( | 314 audio_repetition_detector_.reset( |
| 313 new AudioRepetitionDetector(kMinLengthMs, kMaxFrames, look_back_times, | 315 new AudioRepetitionDetector(kMinLengthMs, kMaxFrames, look_back_times, |
| 314 base::Bind(&ReportRepetition))); | 316 base::Bind(&ReportRepetition))); |
| 315 } | 317 } |
| 316 | 318 |
| 317 MediaStreamAudioProcessor::~MediaStreamAudioProcessor() { | 319 MediaStreamAudioProcessor::~MediaStreamAudioProcessor() { |
| 318 // TODO(miu): This class is ref-counted, shared among threads, and then | 320 // TODO(miu): This class is ref-counted, shared among threads, and then |
| 319 // requires itself to be destroyed on the main thread only?!?!? Fix this, and | 321 // requires itself to be destroyed on the main thread only?!?!? Fix this, and |
| 320 // then remove the hack in WebRtcAudioSink::Adapter. | 322 // then remove the hack in WebRtcAudioSink::Adapter. |
| 321 DCHECK(main_thread_checker_.CalledOnValidThread()); | 323 DCHECK(main_thread_runner_->BelongsToCurrentThread()); |
| 322 Stop(); | 324 Stop(); |
| 323 } | 325 } |
| 324 | 326 |
| 325 void MediaStreamAudioProcessor::OnCaptureFormatChanged( | 327 void MediaStreamAudioProcessor::OnCaptureFormatChanged( |
| 326 const media::AudioParameters& input_format) { | 328 const media::AudioParameters& input_format) { |
| 327 DCHECK(main_thread_checker_.CalledOnValidThread()); | 329 DCHECK(main_thread_runner_->BelongsToCurrentThread()); |
| 330 | |
| 328 // There is no need to hold a lock here since the caller guarantees that | 331 // There is no need to hold a lock here since the caller guarantees that |
| 329 // there is no more PushCaptureData() and ProcessAndConsumeData() callbacks | 332 // there is no more PushCaptureData() and ProcessAndConsumeData() callbacks |
| 330 // on the capture thread. | 333 // on the capture thread. |
| 331 InitializeCaptureFifo(input_format); | 334 InitializeCaptureFifo(input_format); |
| 332 | 335 |
| 333 // Reset the |capture_thread_checker_| since the capture data will come from | 336 // Reset the |capture_thread_checker_| since the capture data will come from |
| 334 // a new capture thread. | 337 // a new capture thread. |
| 335 capture_thread_checker_.DetachFromThread(); | 338 capture_thread_checker_.DetachFromThread(); |
| 336 } | 339 } |
| 337 | 340 |
| (...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 383 // Swap the first and second channels. | 386 // Swap the first and second channels. |
| 384 output_bus->bus()->SwapChannels(0, 1); | 387 output_bus->bus()->SwapChannels(0, 1); |
| 385 } | 388 } |
| 386 | 389 |
| 387 *processed_data = output_bus->bus(); | 390 *processed_data = output_bus->bus(); |
| 388 | 391 |
| 389 return true; | 392 return true; |
| 390 } | 393 } |
| 391 | 394 |
| 392 void MediaStreamAudioProcessor::Stop() { | 395 void MediaStreamAudioProcessor::Stop() { |
| 393 DCHECK(main_thread_checker_.CalledOnValidThread()); | 396 DCHECK(main_thread_runner_->BelongsToCurrentThread()); |
| 394 | 397 |
| 395 if (stopped_) | 398 if (stopped_) |
| 396 return; | 399 return; |
| 397 | 400 |
| 398 stopped_ = true; | 401 stopped_ = true; |
| 399 | 402 |
| 400 if (aec_dump_message_filter_.get()) { | 403 if (aec_dump_message_filter_.get()) { |
| 401 aec_dump_message_filter_->RemoveDelegate(this); | 404 aec_dump_message_filter_->RemoveDelegate(this); |
| 402 aec_dump_message_filter_ = NULL; | 405 aec_dump_message_filter_ = NULL; |
| 403 } | 406 } |
| (...skipping 16 matching lines...) Expand all Loading... | |
| 420 const media::AudioParameters& MediaStreamAudioProcessor::InputFormat() const { | 423 const media::AudioParameters& MediaStreamAudioProcessor::InputFormat() const { |
| 421 return input_format_; | 424 return input_format_; |
| 422 } | 425 } |
| 423 | 426 |
| 424 const media::AudioParameters& MediaStreamAudioProcessor::OutputFormat() const { | 427 const media::AudioParameters& MediaStreamAudioProcessor::OutputFormat() const { |
| 425 return output_format_; | 428 return output_format_; |
| 426 } | 429 } |
| 427 | 430 |
| 428 void MediaStreamAudioProcessor::OnAecDumpFile( | 431 void MediaStreamAudioProcessor::OnAecDumpFile( |
| 429 const IPC::PlatformFileForTransit& file_handle) { | 432 const IPC::PlatformFileForTransit& file_handle) { |
| 430 DCHECK(main_thread_checker_.CalledOnValidThread()); | 433 DCHECK(main_thread_runner_->BelongsToCurrentThread()); |
| 431 | 434 |
| 432 base::File file = IPC::PlatformFileForTransitToFile(file_handle); | 435 base::File file = IPC::PlatformFileForTransitToFile(file_handle); |
| 433 DCHECK(file.IsValid()); | 436 DCHECK(file.IsValid()); |
| 434 | 437 |
| 435 if (audio_processing_) | 438 if (audio_processing_) |
| 436 StartEchoCancellationDump(audio_processing_.get(), std::move(file)); | 439 StartEchoCancellationDump(audio_processing_.get(), std::move(file)); |
| 437 else | 440 else |
| 438 file.Close(); | 441 file.Close(); |
| 439 } | 442 } |
| 440 | 443 |
| 441 void MediaStreamAudioProcessor::OnDisableAecDump() { | 444 void MediaStreamAudioProcessor::OnDisableAecDump() { |
| 442 DCHECK(main_thread_checker_.CalledOnValidThread()); | 445 DCHECK(main_thread_runner_->BelongsToCurrentThread()); |
| 443 if (audio_processing_) | 446 if (audio_processing_) |
| 444 StopEchoCancellationDump(audio_processing_.get()); | 447 StopEchoCancellationDump(audio_processing_.get()); |
| 445 } | 448 } |
| 446 | 449 |
| 447 void MediaStreamAudioProcessor::OnIpcClosing() { | 450 void MediaStreamAudioProcessor::OnIpcClosing() { |
| 448 DCHECK(main_thread_checker_.CalledOnValidThread()); | 451 DCHECK(main_thread_runner_->BelongsToCurrentThread()); |
| 449 aec_dump_message_filter_ = NULL; | 452 aec_dump_message_filter_ = NULL; |
| 450 } | 453 } |
| 451 | 454 |
| 452 // static | 455 // static |
| 453 bool MediaStreamAudioProcessor::WouldModifyAudio( | 456 bool MediaStreamAudioProcessor::WouldModifyAudio( |
| 454 const blink::WebMediaConstraints& constraints, | 457 const blink::WebMediaConstraints& constraints, |
| 455 int effects_flags) { | 458 int effects_flags) { |
| 456 // Note: This method should by kept in-sync with any changes to the logic in | 459 // Note: This method should by kept in-sync with any changes to the logic in |
| 457 // MediaStreamAudioProcessor::InitializeAudioProcessingModule(). | 460 // MediaStreamAudioProcessor::InitializeAudioProcessingModule(). |
| 458 | 461 |
| (...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 513 // TODO(ajm): Should AnalyzeReverseStream() account for the |audio_delay|? | 516 // TODO(ajm): Should AnalyzeReverseStream() account for the |audio_delay|? |
| 514 audio_processing_->AnalyzeReverseStream( | 517 audio_processing_->AnalyzeReverseStream( |
| 515 analysis_bus->channel_ptrs(), | 518 analysis_bus->channel_ptrs(), |
| 516 analysis_bus->bus()->frames(), | 519 analysis_bus->bus()->frames(), |
| 517 sample_rate, | 520 sample_rate, |
| 518 ChannelsToLayout(audio_bus->channels())); | 521 ChannelsToLayout(audio_bus->channels())); |
| 519 } | 522 } |
| 520 } | 523 } |
| 521 | 524 |
| 522 void MediaStreamAudioProcessor::OnPlayoutDataSourceChanged() { | 525 void MediaStreamAudioProcessor::OnPlayoutDataSourceChanged() { |
| 523 DCHECK(main_thread_checker_.CalledOnValidThread()); | 526 DCHECK(main_thread_runner_->BelongsToCurrentThread()); |
| 524 // There is no need to hold a lock here since the caller guarantees that | 527 // There is no need to hold a lock here since the caller guarantees that |
| 525 // there is no more OnPlayoutData() callback on the render thread. | 528 // there is no more OnPlayoutData() callback on the render thread. |
| 526 render_thread_checker_.DetachFromThread(); | 529 render_thread_checker_.DetachFromThread(); |
| 527 render_fifo_.reset(); | 530 render_fifo_.reset(); |
| 528 } | 531 } |
| 529 | 532 |
| 530 void MediaStreamAudioProcessor::OnRenderThreadChanged() { | 533 void MediaStreamAudioProcessor::OnRenderThreadChanged() { |
| 531 render_thread_checker_.DetachFromThread(); | 534 render_thread_checker_.DetachFromThread(); |
| 532 DCHECK(render_thread_checker_.CalledOnValidThread()); | 535 DCHECK(render_thread_checker_.CalledOnValidThread()); |
| 533 render_fifo_->ReattachThreadChecker(); | 536 render_fifo_->ReattachThreadChecker(); |
| 534 } | 537 } |
| 535 | 538 |
| 536 void MediaStreamAudioProcessor::GetStats(AudioProcessorStats* stats) { | 539 void MediaStreamAudioProcessor::GetStats(AudioProcessorStats* stats) { |
| 537 stats->typing_noise_detected = | 540 stats->typing_noise_detected = |
| 538 (base::subtle::Acquire_Load(&typing_detected_) != false); | 541 (base::subtle::Acquire_Load(&typing_detected_) != false); |
| 539 GetAecStats(audio_processing_.get()->echo_cancellation(), stats); | 542 GetAecStats(audio_processing_.get()->echo_cancellation(), stats); |
| 540 } | 543 } |
| 541 | 544 |
| 542 void MediaStreamAudioProcessor::InitializeAudioProcessingModule( | 545 void MediaStreamAudioProcessor::InitializeAudioProcessingModule( |
| 543 const blink::WebMediaConstraints& constraints, | 546 const blink::WebMediaConstraints& constraints, |
| 544 const MediaStreamDevice::AudioDeviceParameters& input_params) { | 547 const MediaStreamDevice::AudioDeviceParameters& input_params) { |
| 545 DCHECK(main_thread_checker_.CalledOnValidThread()); | 548 DCHECK(main_thread_runner_->BelongsToCurrentThread()); |
| 546 DCHECK(!audio_processing_); | 549 DCHECK(!audio_processing_); |
| 547 | 550 |
| 548 MediaAudioConstraints audio_constraints(constraints, input_params.effects); | 551 MediaAudioConstraints audio_constraints(constraints, input_params.effects); |
| 549 | 552 |
| 550 // Note: The audio mirroring constraint (i.e., swap left and right channels) | 553 // Note: The audio mirroring constraint (i.e., swap left and right channels) |
| 551 // is handled within this MediaStreamAudioProcessor and does not, by itself, | 554 // is handled within this MediaStreamAudioProcessor and does not, by itself, |
| 552 // require webrtc::AudioProcessing. | 555 // require webrtc::AudioProcessing. |
| 553 audio_mirroring_ = audio_constraints.GetGoogAudioMirroring(); | 556 audio_mirroring_ = audio_constraints.GetGoogAudioMirroring(); |
| 554 | 557 |
| 555 const bool echo_cancellation = | 558 const bool echo_cancellation = |
| (...skipping 98 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 654 } | 657 } |
| 655 | 658 |
| 656 if (goog_agc) | 659 if (goog_agc) |
| 657 EnableAutomaticGainControl(audio_processing_.get()); | 660 EnableAutomaticGainControl(audio_processing_.get()); |
| 658 | 661 |
| 659 RecordProcessingState(AUDIO_PROCESSING_ENABLED); | 662 RecordProcessingState(AUDIO_PROCESSING_ENABLED); |
| 660 } | 663 } |
| 661 | 664 |
| 662 void MediaStreamAudioProcessor::InitializeCaptureFifo( | 665 void MediaStreamAudioProcessor::InitializeCaptureFifo( |
| 663 const media::AudioParameters& input_format) { | 666 const media::AudioParameters& input_format) { |
| 664 DCHECK(main_thread_checker_.CalledOnValidThread()); | 667 DCHECK(main_thread_runner_->BelongsToCurrentThread()); |
| 665 DCHECK(input_format.IsValid()); | 668 DCHECK(input_format.IsValid()); |
| 666 input_format_ = input_format; | 669 input_format_ = input_format; |
| 667 | 670 |
| 668 // TODO(ajm): For now, we assume fixed parameters for the output when audio | 671 // TODO(ajm): For now, we assume fixed parameters for the output when audio |
| 669 // processing is enabled, to match the previous behavior. We should either | 672 // processing is enabled, to match the previous behavior. We should either |
| 670 // use the input parameters (in which case, audio processing will convert | 673 // use the input parameters (in which case, audio processing will convert |
| 671 // at output) or ideally, have a backchannel from the sink to know what | 674 // at output) or ideally, have a backchannel from the sink to know what |
| 672 // format it would prefer. | 675 // format it would prefer. |
| 673 #if defined(OS_ANDROID) | 676 #if defined(OS_ANDROID) |
| 674 int audio_processing_sample_rate = AudioProcessing::kSampleRate16kHz; | 677 int audio_processing_sample_rate = AudioProcessing::kSampleRate16kHz; |
| (...skipping 127 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 802 DCHECK_EQ(err, 0) << "ProcessStream() error: " << err; | 805 DCHECK_EQ(err, 0) << "ProcessStream() error: " << err; |
| 803 | 806 |
| 804 if (typing_detector_) { | 807 if (typing_detector_) { |
| 805 webrtc::VoiceDetection* vad = ap->voice_detection(); | 808 webrtc::VoiceDetection* vad = ap->voice_detection(); |
| 806 DCHECK(vad->is_enabled()); | 809 DCHECK(vad->is_enabled()); |
| 807 bool detected = typing_detector_->Process(key_pressed, | 810 bool detected = typing_detector_->Process(key_pressed, |
| 808 vad->stream_has_voice()); | 811 vad->stream_has_voice()); |
| 809 base::subtle::Release_Store(&typing_detected_, detected); | 812 base::subtle::Release_Store(&typing_detected_, detected); |
| 810 } | 813 } |
| 811 | 814 |
| 812 main_thread_message_loop_->task_runner()->PostTask( | 815 main_thread_runner_->PostTask( |
|
miu
2016/10/19 02:06:40
Note: This LOC is where the data race occurred.
| |
| 813 FROM_HERE, base::Bind(&MediaStreamAudioProcessor::UpdateAecStats, this)); | 816 FROM_HERE, base::Bind(&MediaStreamAudioProcessor::UpdateAecStats, this)); |
| 814 | 817 |
| 815 // Return 0 if the volume hasn't been changed, and otherwise the new volume. | 818 // Return 0 if the volume hasn't been changed, and otherwise the new volume. |
| 816 return (agc->stream_analog_level() == volume) ? | 819 return (agc->stream_analog_level() == volume) ? |
| 817 0 : agc->stream_analog_level(); | 820 0 : agc->stream_analog_level(); |
| 818 } | 821 } |
| 819 | 822 |
| 820 void MediaStreamAudioProcessor::UpdateAecStats() { | 823 void MediaStreamAudioProcessor::UpdateAecStats() { |
| 821 DCHECK(main_thread_checker_.CalledOnValidThread()); | 824 DCHECK(main_thread_runner_->BelongsToCurrentThread()); |
| 822 if (echo_information_) | 825 if (echo_information_) |
| 823 echo_information_->UpdateAecStats(audio_processing_->echo_cancellation()); | 826 echo_information_->UpdateAecStats(audio_processing_->echo_cancellation()); |
| 824 } | 827 } |
| 825 | 828 |
| 826 } // namespace content | 829 } // namespace content |
| OLD | NEW |