1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229
|
/*
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_AUDIO_PROCESSING_AEC3_ECHO_CANCELLER3_H_
#define MODULES_AUDIO_PROCESSING_AEC3_ECHO_CANCELLER3_H_
#include <stddef.h>
#include <atomic>
#include <memory>
#include <optional>
#include <vector>
#include "api/array_view.h"
#include "api/audio/echo_canceller3_config.h"
#include "api/audio/echo_control.h"
#include "modules/audio_processing/aec3/api_call_jitter_metrics.h"
#include "modules/audio_processing/aec3/block_delay_buffer.h"
#include "modules/audio_processing/aec3/block_framer.h"
#include "modules/audio_processing/aec3/block_processor.h"
#include "modules/audio_processing/aec3/config_selector.h"
#include "modules/audio_processing/aec3/frame_blocker.h"
#include "modules/audio_processing/aec3/multi_channel_content_detector.h"
#include "modules/audio_processing/audio_buffer.h"
#include "modules/audio_processing/logging/apm_data_dumper.h"
#include "rtc_base/checks.h"
#include "rtc_base/race_checker.h"
#include "rtc_base/swap_queue.h"
#include "rtc_base/thread_annotations.h"
namespace webrtc {
// Method for adjusting config parameter dependencies.
// Only to be used externally to AEC3 for testing purposes.
// TODO(webrtc:5298): Move this to a separate file.
EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config);
// Functor for verifying the invariance of the frames being put into the render
// queue.
class Aec3RenderQueueItemVerifier {
public:
Aec3RenderQueueItemVerifier(size_t num_bands,
size_t num_channels,
size_t frame_length)
: num_bands_(num_bands),
num_channels_(num_channels),
frame_length_(frame_length) {}
bool operator()(const std::vector<std::vector<std::vector<float>>>& v) const {
if (v.size() != num_bands_) {
return false;
}
for (const auto& band : v) {
if (band.size() != num_channels_) {
return false;
}
for (const auto& channel : band) {
if (channel.size() != frame_length_) {
return false;
}
}
}
return true;
}
private:
const size_t num_bands_;
const size_t num_channels_;
const size_t frame_length_;
};
// Main class for the echo canceller3.
// It does 4 things:
// -Receives 10 ms frames of band-split audio.
// -Provides the lower level echo canceller functionality with
// blocks of 64 samples of audio data.
// -Partially handles the jitter in the render and capture API
// call sequence.
//
// The class is supposed to be used in a non-concurrent manner apart from the
// AnalyzeRender call which can be called concurrently with the other methods.
class EchoCanceller3 : public EchoControl {
public:
EchoCanceller3(const EchoCanceller3Config& config,
const std::optional<EchoCanceller3Config>& multichannel_config,
int sample_rate_hz,
size_t num_render_channels,
size_t num_capture_channels);
~EchoCanceller3() override;
EchoCanceller3(const EchoCanceller3&) = delete;
EchoCanceller3& operator=(const EchoCanceller3&) = delete;
// Analyzes and stores an internal copy of the split-band domain render
// signal.
void AnalyzeRender(AudioBuffer* render) override { AnalyzeRender(*render); }
// Analyzes the full-band domain capture signal to detect signal saturation.
void AnalyzeCapture(AudioBuffer* capture) override {
AnalyzeCapture(*capture);
}
// Processes the split-band domain capture signal in order to remove any echo
// present in the signal.
void ProcessCapture(AudioBuffer* capture, bool level_change) override;
// As above, but also returns the linear filter output.
void ProcessCapture(AudioBuffer* capture,
AudioBuffer* linear_output,
bool level_change) override;
// Collect current metrics from the echo canceller.
Metrics GetMetrics() const override;
// Provides an optional external estimate of the audio buffer delay.
void SetAudioBufferDelay(int delay_ms) override;
// Specifies whether the capture output will be used. The purpose of this is
// to allow the echo controller to deactivate some of the processing when the
// resulting output is anyway not used, for instance when the endpoint is
// muted.
void SetCaptureOutputUsage(bool capture_output_used) override;
bool ActiveProcessing() const override;
// Signals whether an external detector has detected echo leakage from the
// echo canceller.
// Note that in the case echo leakage has been flagged, it should be unflagged
// once it is no longer occurring.
void UpdateEchoLeakageStatus(bool leakage_detected) {
RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_);
block_processor_->UpdateEchoLeakageStatus(leakage_detected);
}
// Produces a default configuration for multichannel.
static EchoCanceller3Config CreateDefaultMultichannelConfig();
private:
friend class EchoCanceller3Tester;
FRIEND_TEST_ALL_PREFIXES(EchoCanceller3, DetectionOfProperStereo);
FRIEND_TEST_ALL_PREFIXES(EchoCanceller3,
DetectionOfProperStereoUsingThreshold);
FRIEND_TEST_ALL_PREFIXES(EchoCanceller3,
DetectionOfProperStereoUsingHysteresis);
FRIEND_TEST_ALL_PREFIXES(EchoCanceller3,
StereoContentDetectionForMonoSignals);
class RenderWriter;
// (Re-)Initializes the selected subset of the EchoCanceller3 fields, at
// creation as well as during reconfiguration.
void Initialize();
// Only for testing. Replaces the internal block processor.
void SetBlockProcessorForTesting(
std::unique_ptr<BlockProcessor> block_processor);
// Only for testing. Returns whether stereo processing is active.
bool StereoRenderProcessingActiveForTesting() const {
return multichannel_content_detector_.IsProperMultiChannelContentDetected();
}
// Only for testing.
const EchoCanceller3Config& GetActiveConfigForTesting() const {
return config_selector_.active_config();
}
// Empties the render SwapQueue.
void EmptyRenderQueue();
// Analyzes and stores an internal copy of the split-band domain render
// signal.
void AnalyzeRender(const AudioBuffer& render);
// Analyzes the full-band domain capture signal to detect signal saturation.
void AnalyzeCapture(const AudioBuffer& capture);
rtc::RaceChecker capture_race_checker_;
rtc::RaceChecker render_race_checker_;
// State that is accessed by the AnalyzeRender call.
std::unique_ptr<RenderWriter> render_writer_
RTC_GUARDED_BY(render_race_checker_);
// State that may be accessed by the capture thread.
static std::atomic<int> instance_count_;
std::unique_ptr<ApmDataDumper> data_dumper_;
const EchoCanceller3Config config_;
const int sample_rate_hz_;
const int num_bands_;
const size_t num_render_input_channels_;
size_t num_render_channels_to_aec_;
const size_t num_capture_channels_;
ConfigSelector config_selector_;
MultiChannelContentDetector multichannel_content_detector_;
std::unique_ptr<BlockFramer> linear_output_framer_
RTC_GUARDED_BY(capture_race_checker_);
BlockFramer output_framer_ RTC_GUARDED_BY(capture_race_checker_);
FrameBlocker capture_blocker_ RTC_GUARDED_BY(capture_race_checker_);
std::unique_ptr<FrameBlocker> render_blocker_
RTC_GUARDED_BY(capture_race_checker_);
SwapQueue<std::vector<std::vector<std::vector<float>>>,
Aec3RenderQueueItemVerifier>
render_transfer_queue_;
std::unique_ptr<BlockProcessor> block_processor_
RTC_GUARDED_BY(capture_race_checker_);
std::vector<std::vector<std::vector<float>>> render_queue_output_frame_
RTC_GUARDED_BY(capture_race_checker_);
bool saturated_microphone_signal_ RTC_GUARDED_BY(capture_race_checker_) =
false;
Block render_block_ RTC_GUARDED_BY(capture_race_checker_);
std::unique_ptr<Block> linear_output_block_
RTC_GUARDED_BY(capture_race_checker_);
Block capture_block_ RTC_GUARDED_BY(capture_race_checker_);
std::vector<std::vector<rtc::ArrayView<float>>> render_sub_frame_view_
RTC_GUARDED_BY(capture_race_checker_);
std::vector<std::vector<rtc::ArrayView<float>>> linear_output_sub_frame_view_
RTC_GUARDED_BY(capture_race_checker_);
std::vector<std::vector<rtc::ArrayView<float>>> capture_sub_frame_view_
RTC_GUARDED_BY(capture_race_checker_);
std::unique_ptr<BlockDelayBuffer> block_delay_buffer_
RTC_GUARDED_BY(capture_race_checker_);
ApiCallJitterMetrics api_call_metrics_ RTC_GUARDED_BY(capture_race_checker_);
};
} // namespace webrtc
#endif // MODULES_AUDIO_PROCESSING_AEC3_ECHO_CANCELLER3_H_
|