1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
|
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_AUDIO_PROCESSING_AEC3_AEC_STATE_H_
#define MODULES_AUDIO_PROCESSING_AEC3_AEC_STATE_H_
#include <stddef.h>
#include <array>
#include <atomic>
#include <memory>
#include <optional>
#include <vector>
#include "api/array_view.h"
#include "api/audio/echo_canceller3_config.h"
#include "api/environment/environment.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/delay_estimate.h"
#include "modules/audio_processing/aec3/echo_audibility.h"
#include "modules/audio_processing/aec3/echo_path_variability.h"
#include "modules/audio_processing/aec3/erl_estimator.h"
#include "modules/audio_processing/aec3/erle_estimator.h"
#include "modules/audio_processing/aec3/filter_analyzer.h"
#include "modules/audio_processing/aec3/render_buffer.h"
#include "modules/audio_processing/aec3/reverb_model_estimator.h"
#include "modules/audio_processing/aec3/subtractor_output.h"
#include "modules/audio_processing/aec3/subtractor_output_analyzer.h"
#include "modules/audio_processing/aec3/transparent_mode.h"
namespace webrtc {
class ApmDataDumper;
// Handles the state and the conditions for the echo removal functionality.
class AecState {
public:
AecState(const Environment& env,
const EchoCanceller3Config& config,
size_t num_capture_channels);
~AecState();
// Returns whether the echo subtractor can be used to determine the residual
// echo.
bool UsableLinearEstimate() const {
return filter_quality_state_.LinearFilterUsable() &&
config_.filter.use_linear_filter;
}
// Returns whether the echo subtractor output should be used as output.
bool UseLinearFilterOutput() const {
return filter_quality_state_.LinearFilterUsable() &&
config_.filter.use_linear_filter;
}
// Returns whether the render signal is currently active.
bool ActiveRender() const { return blocks_with_active_render_ > 200; }
// Returns the appropriate scaling of the residual echo to match the
// audibility.
void GetResidualEchoScaling(ArrayView<float> residual_scaling) const;
// Returns whether the stationary properties of the signals are used in the
// aec.
bool UseStationarityProperties() const {
return config_.echo_audibility.use_stationarity_properties;
}
// Returns the ERLE.
ArrayView<const std::array<float, kFftLengthBy2Plus1>> Erle(
bool onset_compensated) const {
return erle_estimator_.Erle(onset_compensated);
}
// Returns the non-capped ERLE.
ArrayView<const std::array<float, kFftLengthBy2Plus1>> ErleUnbounded() const {
return erle_estimator_.ErleUnbounded();
}
// Returns the fullband ERLE estimate in log2 units.
float FullBandErleLog2() const { return erle_estimator_.FullbandErleLog2(); }
// Returns the ERL.
const std::array<float, kFftLengthBy2Plus1>& Erl() const {
return erl_estimator_.Erl();
}
// Returns the time-domain ERL.
float ErlTimeDomain() const { return erl_estimator_.ErlTimeDomain(); }
// Returns the delay estimate based on the linear filter.
int MinDirectPathFilterDelay() const {
return delay_state_.MinDirectPathFilterDelay();
}
// Returns whether the capture signal is saturated.
bool SaturatedCapture() const { return capture_signal_saturation_; }
// Returns whether the echo signal is saturated.
bool SaturatedEcho() const { return saturation_detector_.SaturatedEcho(); }
// Updates the capture signal saturation.
void UpdateCaptureSaturation(bool capture_signal_saturation) {
capture_signal_saturation_ = capture_signal_saturation;
}
// Returns whether the transparent mode is active
bool TransparentModeActive() const {
return transparent_state_ && transparent_state_->Active();
}
// Takes appropriate action at an echo path change.
void HandleEchoPathChange(const EchoPathVariability& echo_path_variability);
// Returns the decay factor for the echo reverberation. The parameter `mild`
// indicates which exponential decay to return. The default one or a milder
// one that can be used during nearend regions.
float ReverbDecay(bool mild) const {
return reverb_model_estimator_.ReverbDecay(mild);
}
// Return the frequency response of the reverberant echo.
ArrayView<const float> GetReverbFrequencyResponse() const {
return reverb_model_estimator_.GetReverbFrequencyResponse();
}
// Returns whether the transition for going out of the initial stated has
// been triggered.
bool TransitionTriggered() const {
return initial_state_.TransitionTriggered();
}
// Updates the aec state.
// TODO(bugs.webrtc.org/10913): Compute multi-channel ERL.
void Update(
const std::optional<DelayEstimate>& external_delay,
ArrayView<const std::vector<std::array<float, kFftLengthBy2Plus1>>>
adaptive_filter_frequency_responses,
ArrayView<const std::vector<float>> adaptive_filter_impulse_responses,
const RenderBuffer& render_buffer,
ArrayView<const std::array<float, kFftLengthBy2Plus1>> E2_refined,
ArrayView<const std::array<float, kFftLengthBy2Plus1>> Y2,
ArrayView<const SubtractorOutput> subtractor_output);
// Returns filter length in blocks.
int FilterLengthBlocks() const {
// All filters have the same length, so arbitrarily return channel 0 length.
return filter_analyzer_.FilterLengthBlocks();
}
private:
static std::atomic<int> instance_count_;
std::unique_ptr<ApmDataDumper> data_dumper_;
const EchoCanceller3Config config_;
const size_t num_capture_channels_;
const bool deactivate_initial_state_reset_at_echo_path_change_;
const bool full_reset_at_echo_path_change_;
const bool subtractor_analyzer_reset_at_echo_path_change_;
// Class for controlling the transition from the intial state, which in turn
// controls when the filter parameters for the initial state should be used.
class InitialState {
public:
explicit InitialState(const EchoCanceller3Config& config);
// Resets the state to again begin in the initial state.
void Reset();
// Updates the state based on new data.
void Update(bool active_render, bool saturated_capture);
// Returns whether the initial state is active or not.
bool InitialStateActive() const { return initial_state_; }
// Returns that the transition from the initial state has was started.
bool TransitionTriggered() const { return transition_triggered_; }
private:
const bool conservative_initial_phase_;
const float initial_state_seconds_;
bool transition_triggered_ = false;
bool initial_state_ = true;
size_t strong_not_saturated_render_blocks_ = 0;
} initial_state_;
// Class for choosing the direct-path delay relative to the beginning of the
// filter, as well as any other data related to the delay used within
// AecState.
class FilterDelay {
public:
FilterDelay(const EchoCanceller3Config& config,
size_t num_capture_channels);
// Returns whether an external delay has been reported to the AecState (from
// the delay estimator).
bool ExternalDelayReported() const { return external_delay_reported_; }
// Returns the delay in blocks relative to the beginning of the filter that
// corresponds to the direct path of the echo.
ArrayView<const int> DirectPathFilterDelays() const {
return filter_delays_blocks_;
}
// Returns the minimum delay among the direct path delays relative to the
// beginning of the filter
int MinDirectPathFilterDelay() const { return min_filter_delay_; }
// Updates the delay estimates based on new data.
void Update(ArrayView<const int> analyzer_filter_delay_estimates_blocks,
const std::optional<DelayEstimate>& external_delay,
size_t blocks_with_proper_filter_adaptation);
private:
const int delay_headroom_blocks_;
bool external_delay_reported_ = false;
std::vector<int> filter_delays_blocks_;
int min_filter_delay_;
std::optional<DelayEstimate> external_delay_;
} delay_state_;
// Classifier for toggling transparent mode when there is no echo.
std::unique_ptr<TransparentMode> transparent_state_;
// Class for analyzing how well the linear filter is, and can be expected to,
// perform on the current signals. The purpose of this is for using to
// select the echo suppression functionality as well as the input to the echo
// suppressor.
class FilteringQualityAnalyzer {
public:
FilteringQualityAnalyzer(const EchoCanceller3Config& config,
size_t num_capture_channels);
// Returns whether the linear filter can be used for the echo
// canceller output.
bool LinearFilterUsable() const { return overall_usable_linear_estimates_; }
// Returns whether an individual filter output can be used for the echo
// canceller output.
const std::vector<bool>& UsableLinearFilterOutputs() const {
return usable_linear_filter_estimates_;
}
// Resets the state of the analyzer.
void Reset();
// Updates the analysis based on new data.
void Update(bool active_render,
bool transparent_mode,
bool saturated_capture,
const std::optional<DelayEstimate>& external_delay,
bool any_filter_converged);
private:
const bool use_linear_filter_;
bool overall_usable_linear_estimates_ = false;
size_t filter_update_blocks_since_reset_ = 0;
size_t filter_update_blocks_since_start_ = 0;
bool convergence_seen_ = false;
std::vector<bool> usable_linear_filter_estimates_;
} filter_quality_state_;
// Class for detecting whether the echo is to be considered to be
// saturated.
class SaturationDetector {
public:
// Returns whether the echo is to be considered saturated.
bool SaturatedEcho() const { return saturated_echo_; }
// Updates the detection decision based on new data.
void Update(const Block& x,
bool saturated_capture,
bool usable_linear_estimate,
ArrayView<const SubtractorOutput> subtractor_output,
float echo_path_gain);
private:
bool saturated_echo_ = false;
} saturation_detector_;
ErlEstimator erl_estimator_;
ErleEstimator erle_estimator_;
size_t strong_not_saturated_render_blocks_ = 0;
size_t blocks_with_active_render_ = 0;
bool capture_signal_saturation_ = false;
FilterAnalyzer filter_analyzer_;
EchoAudibility echo_audibility_;
ReverbModelEstimator reverb_model_estimator_;
ReverbModel avg_render_reverb_;
SubtractorOutputAnalyzer subtractor_output_analyzer_;
};
} // namespace webrtc
#endif // MODULES_AUDIO_PROCESSING_AEC3_AEC_STATE_H_
|