File: input_controller.h

package info (click to toggle)
chromium 145.0.7632.159-1
  • links: PTS, VCS
  • area: main
  • in suites: sid
  • size: 5,976,224 kB
  • sloc: cpp: 36,198,469; ansic: 7,634,080; javascript: 3,564,060; python: 1,649,622; xml: 838,470; asm: 717,087; pascal: 185,708; sh: 88,786; perl: 88,718; objc: 79,984; sql: 59,811; cs: 42,452; fortran: 24,101; makefile: 21,144; tcl: 15,277; php: 14,022; yacc: 9,066; ruby: 7,553; awk: 3,720; lisp: 3,233; lex: 1,328; ada: 727; jsp: 228; sed: 36
file content (411 lines) | stat: -rw-r--r-- 15,380 bytes parent folder | download | duplicates (3)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
// Copyright 2012 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifndef SERVICES_AUDIO_INPUT_CONTROLLER_H_
#define SERVICES_AUDIO_INPUT_CONTROLLER_H_

#include <stddef.h>
#include <stdint.h>

#include <memory>
#include <optional>
#include <string>
#include <string_view>

#include "base/memory/raw_ptr.h"
#include "base/memory/weak_ptr.h"
#include "base/task/single_thread_task_runner.h"
#include "base/threading/thread_checker.h"
#include "base/time/time.h"
#include "base/timer/timer.h"
#include "build/build_config.h"
#include "media/base/audio_parameters.h"
#include "media/base/audio_processing.h"
#include "media/media_buildflags.h"
#include "media/mojo/mojom/audio_processing.mojom.h"
#include "services/audio/loopback_mixin.h"

namespace media {
class AecdumpRecordingManager;
class AudioBus;
class AudioInputStream;
class AudioManager;
struct AudioGlitchInfo;
}  // namespace media

namespace audio {
class AudioProcessorHandler;
class AudioCallback;
class MlModelManager;
class OutputTapper;
class ReferenceSignalProvider;

#if BUILDFLAG(CHROME_WIDE_ECHO_CANCELLATION)
class ProcessingAudioFifo;
#endif

// Only do power monitoring for non-mobile platforms to save resources.
#if !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS)
#define AUDIO_POWER_MONITORING
#endif

// All public methods of InputController must be called from the audio thread.
//
// Audio data flow through InputController:
//
// * Without any audio processing:
//     InputController::|audio_callback_|::OnData()
//     -> InputController::OnData()
//     --> InputController::|sync_writer_|::Write()
//
// * With audio processing but no dedicated processing thread:
//     InputController::|audio_callback_|::OnData()
//     -> InputController::OnData()
//     --> InputController::|audio_processor_handler_|::ProcessCapturedAudio()
//     ---> InputController::DeliverProcessedAudio()
//     ----> InputController::|sync_writer_|::Write()
//
// * With audio processing and a dedicated processing thread:
//   Audio capture device thread:
//     InputController::|audio_callback_|::OnData()
//     -> InputController::OnData()
//     --> InputController::|processing_fifo_|::PushData()
//   Audio processing thread:
//     ---> InputController::|audio_processor_handler_|::ProcessCapturedAudio()
//     ----> InputController::DeliverProcessedAudio()
//     -----> InputController::|sync_writer_|::Write()
//
//     - InputController::|audio_processor_handler_| changes format from the
//     AudioInputStream format to |params| provided to
//     InputController::Create().
//
class InputController final {
 public:
  // Error codes to make native logging more clear. These error codes are added
  // to generic error strings to provide a higher degree of details.
  // Changing these values can lead to problems when matching native debug
  // logs with the actual cause of error.
  enum ErrorCode {
    // An unspecified error occurred.
    UNKNOWN_ERROR = 0,

    // Failed to create an audio input stream.
    STREAM_CREATE_ERROR,  // = 1

    // Failed to open an audio input stream.
    STREAM_OPEN_ERROR,  // = 2

    // Native input stream reports an error. Exact reason differs between
    // platforms.
    STREAM_ERROR,  // = 3

    // Open failed due to lack of system permissions.
    STREAM_OPEN_SYSTEM_PERMISSIONS_ERROR,  // = 4

    // Open failed due to device in use by another app.
    STREAM_OPEN_DEVICE_IN_USE_ERROR,  // = 5

    // Native input stream reports an error. Exact reason differs between
    // platforms.
    REFERENCE_STREAM_ERROR,  // = 6

    // Failed to create aec reference stream.
    REFERENCE_STREAM_CREATE_ERROR,  // = 7

    // Failed to open aec reference stream.
    REFERENCE_STREAM_OPEN_ERROR,  // = 8

    // Failed to open aec reference stream due to lack of system permissions.
    REFERENCE_STREAM_OPEN_SYSTEM_PERMISSIONS_ERROR,  // = 9

    // Failed to open aec reference stream due to device in use by another app.
    REFERENCE_STREAM_OPEN_DEVICE_IN_USE_ERROR,  // = 10
  };

#if defined(AUDIO_POWER_MONITORING)
  // Used to log a silence report (see OnData).
  // Elements in this enum should not be deleted or rearranged; the only
  // permitted operation is to add new elements before SILENCE_STATE_MAX and
  // update SILENCE_STATE_MAX.
  // Possible silence state transitions:
  //           SILENCE_STATE_AUDIO_AND_SILENCE
  //               ^                  ^
  // SILENCE_STATE_ONLY_AUDIO   SILENCE_STATE_ONLY_SILENCE
  //               ^                  ^
  //            SILENCE_STATE_NO_MEASUREMENT
  enum SilenceState {
    SILENCE_STATE_NO_MEASUREMENT = 0,
    SILENCE_STATE_ONLY_AUDIO = 1,
    SILENCE_STATE_ONLY_SILENCE = 2,
    SILENCE_STATE_AUDIO_AND_SILENCE = 3,
    SILENCE_STATE_MAX = SILENCE_STATE_AUDIO_AND_SILENCE
  };
#endif

#if BUILDFLAG(CHROME_WIDE_ECHO_CANCELLATION)
  static constexpr int kProcessingFifoSize = 10;
#endif

  // An event handler that receives events from the InputController. The
  // following methods are all called on the audio thread.
  class EventHandler {
   public:
    // The initial "muted" state of the underlying stream is sent along with the
    // OnCreated callback, to avoid the stream being treated as unmuted until an
    // OnMuted callback has had time to be processed.
    virtual void OnCreated(bool initially_muted) = 0;
    virtual void OnError(ErrorCode error_code) = 0;
    virtual void OnLog(std::string_view) = 0;
    // Called whenever the muted state of the underlying stream changes.
    virtual void OnMuted(bool is_muted) = 0;

   protected:
    virtual ~EventHandler() {}
  };

  // A synchronous writer interface used by InputController for
  // synchronous writing.
  class SyncWriter {
   public:
    virtual ~SyncWriter() {}

    // Write certain amount of data from |data|.
    virtual void Write(const media::AudioBus* data,
                       double volume,
                       base::TimeTicks capture_time,
                       const media::AudioGlitchInfo& glitch_info) = 0;

    // Close this synchronous writer.
    virtual void Close() = 0;
  };

  // enum used for determining what UMA stats to report.
  enum StreamType {
    HIGH_LATENCY = 0,
    LOW_LATENCY = 1,
    FAKE = 2,
  };

  InputController(const InputController&) = delete;
  InputController& operator=(const InputController&) = delete;

  ~InputController();

  media::AudioInputStream* stream_for_testing() { return stream_; }

  static std::unique_ptr<InputController> Create(
      media::AudioManager* audio_manager,
      EventHandler* event_handler,
      SyncWriter* sync_writer,
      std::unique_ptr<ReferenceSignalProvider> reference_signal_provider,
      media::AecdumpRecordingManager* aecdump_recording_manager,
      raw_ptr<MlModelManager> ml_model_manager,
      media::mojom::AudioProcessingConfigPtr processing_config,
      LoopbackMixin::MaybeCreateCallback maybe_create_loopback_mixin_cb,
      const media::AudioParameters& params,
      const std::string& device_id,
      bool agc_is_enabled);

  // Starts recording using the created audio input stream.
  void Record();

  // Closes the audio input stream, freeing the associated resources. Must be
  // called before destruction.
  void Close();

  // Sets the capture volume of the input stream. The value 0.0 corresponds
  // to muted and 1.0 to maximum volume.
  void SetVolume(double volume);

  // Sets the output device which will be used to cancel audio from, if this
  // input device supports echo cancellation.
  void SetOutputDeviceForAec(const std::string& output_device_id);

 private:
  class DelayReporter;
  friend class InputControllerTestHelper;

  // Used to log the result of capture startup.
  // This was previously logged as a boolean with only the no callback and OK
  // options. The enum order is kept to ensure backwards compatibility.
  // Elements in this enum should not be deleted or rearranged; the only
  // permitted operation is to add new elements before
  // CAPTURE_STARTUP_RESULT_MAX and update CAPTURE_STARTUP_RESULT_MAX.
  //
  // The NO_DATA_CALLBACK enum has been replaced with NEVER_GOT_DATA,
  // and there are also other histograms such as
  // Media.Audio.InputStartupSuccessMac to cover issues similar
  // to the ones the NO_DATA_CALLBACK was intended for.
  enum CaptureStartupResult {
    CAPTURE_STARTUP_OK = 0,
    CAPTURE_STARTUP_CREATE_STREAM_FAILED = 1,
    CAPTURE_STARTUP_OPEN_STREAM_FAILED = 2,
    CAPTURE_STARTUP_NEVER_GOT_DATA = 3,
    CAPTURE_STARTUP_STOPPED_EARLY = 4,
    CAPTURE_STARTUP_RESULT_MAX = CAPTURE_STARTUP_STOPPED_EARLY,
  };

  InputController(
      EventHandler* event_handler,
      SyncWriter* sync_writer,
      std::unique_ptr<ReferenceSignalProvider> reference_signal_provider,
      media::AecdumpRecordingManager* aecdump_recording_manager,
      raw_ptr<MlModelManager> ml_model_manager,
      media::mojom::AudioProcessingConfigPtr processing_config,
      const media::AudioParameters& output_params,
      const media::AudioParameters& device_params,
      StreamType type);

  void DoCreate(
      media::AudioManager* audio_manager,
      const media::AudioParameters& params,
      const std::string& device_id,
      bool enable_agc,
      LoopbackMixin::MaybeCreateCallback maybe_create_loopback_mixin_cb);
  void DoReportError(ErrorCode error_code);
  void DoLogAudioLevels(float level_dbfs, int microphone_volume_percent);

#if defined(AUDIO_POWER_MONITORING)
  // Updates the silence state, see enum SilenceState above for state
  // transitions.
  void UpdateSilenceState(bool silence);
#endif

  // Logs the result of creating an InputController.
  void LogCaptureStartupResult(CaptureStartupResult result);

  // Logs whether an error was encountered for the native input stream.
  void LogCallbackError();

  // Called by the native input stream with log messages.
  void LogMessage(const std::string& message);

  // Helper method for creating internal log messages prefixed with "AIC::".
  PRINTF_FORMAT(2, 3) void SendLogMessage(const char* format, ...);

  // Does power monitoring on supported platforms.
  // Called on the hw callback thread.
  // Returns true iff average power and mic volume was returned and should
  // be posted to DoLogAudioLevels on the audio thread.
  // Returns false if either power measurements are disabled or aren't needed
  // right now (they're done periodically).
  bool CheckAudioPower(const media::AudioBus* source,
                       double volume,
                       float* average_power_dbfs,
                       int* mic_volume_percent);

  void CheckMutedState();

  // Called once at first audio callback.
  void ReportIsAlive();

  // Receives new input data on the hw callback thread.
  void OnData(const media::AudioBus* source,
              base::TimeTicks capture_time,
              double volume,
              const media::AudioGlitchInfo& glitch_info);

#if BUILDFLAG(CHROME_WIDE_ECHO_CANCELLATION)
  // Called from the constructor. Helper to isolate logic setting up audio
  // processing components.
  void MaybeSetUpAudioProcessing(
      media::mojom::AudioProcessingConfigPtr processing_config,
      const media::AudioParameters& processing_output_params,
      const media::AudioParameters& device_params,
      std::unique_ptr<ReferenceSignalProvider> reference_signal_provider,
      media::AecdumpRecordingManager* aecdump_recording_manager,
      raw_ptr<MlModelManager> ml_model_manager);

  // Used as a callback for |audio_processor_handler_|.
  void DeliverProcessedAudio(const media::AudioBus& audio_bus,
                             base::TimeTicks audio_capture_time,
                             std::optional<double> new_volume,
                             const media::AudioGlitchInfo& glitch_info);
#endif

  static StreamType ParamsToStreamType(const media::AudioParameters& params);

  // The task runner for the audio manager. All control methods should be called
  // via tasks run by this TaskRunner.
  const scoped_refptr<base::SingleThreadTaskRunner> task_runner_;

  // Contains the InputController::EventHandler which receives state
  // notifications from this class.
  const raw_ptr<EventHandler> event_handler_;

  // Pointer to the audio input stream object.
  // Only used on the audio thread.
  raw_ptr<media::AudioInputStream, DanglingUntriaged> stream_ = nullptr;

  // SyncWriter is used only in low-latency mode for synchronous writing.
  const raw_ptr<SyncWriter> sync_writer_;

  StreamType type_;

  // Helper class to report capture delay UMA stats.
  std::unique_ptr<DelayReporter> delay_reporter_;

  double max_volume_ = 0.0;

#if BUILDFLAG(CHROME_WIDE_ECHO_CANCELLATION)
  // Handles audio processing effects applied to the microphone capture audio.
  std::unique_ptr<AudioProcessorHandler> audio_processor_handler_;

  // Offloads processing captured data to its own real time thread.
  // Note: Ordering is important, as |processing_fifo_| must be destroyed before
  // |audio_processing_handler_|.
  std::unique_ptr<ProcessingAudioFifo> processing_fifo_;

  // Manages the |audio_processor_handler_| subscription to output audio.
  std::unique_ptr<OutputTapper> output_tapper_;
#endif

#if defined(AUDIO_POWER_MONITORING)
  // Whether the silence state and microphone levels should be checked and sent
  // as UMA stats.
  bool power_measurement_is_enabled_ = false;

  // Updated each time a power measurement is performed.
  base::TimeTicks last_audio_level_log_time_;

  // The silence report sent as UMA stat at the end of a session.
  SilenceState silence_state_ = SILENCE_STATE_NO_MEASUREMENT;
#endif

  size_t prev_key_down_count_ = 0;

  // Time when the stream started recording.
  base::TimeTicks stream_create_time_;

  bool is_muted_ = false;
  base::RepeatingTimer check_muted_state_timer_;

  // If configured, used to add chromium playout to the captured audio signal.
  std::unique_ptr<LoopbackMixin> loopback_mixin_;

  // Holds a pointer to the callback object that receives audio data from
  // the lower audio layer. Valid only while 'recording' (between calls to
  // stream_->Start() and stream_->Stop()).
  // The value of this pointer is only set and read on the audio thread while
  // the callbacks themselves occur on the hw callback thread. More details
  // in the AudioCallback class in the cc file.
  std::unique_ptr<AudioCallback> audio_callback_;

  // A weak pointer factory that we use when posting tasks to the audio thread
  // that we want to be automatically discarded after Close() has been called
  // and that we do not want to keep the InputController instance alive
  // beyond what is desired by the user of the instance. An example of where
  // this is important is when we fire error notifications from the hw callback
  // thread, post them to the audio thread. In that case, we do not want the
  // error notification to keep the InputController alive for as long as
  // the error notification is pending and then make a callback from an
  // InputController that has already been closed.
  // All outstanding weak pointers are invalidated at the end of Close().
  base::WeakPtr<InputController> weak_this_;
  base::WeakPtrFactory<InputController> weak_ptr_factory_{this};
};

}  // namespace audio

#endif  // SERVICES_AUDIO_INPUT_CONTROLLER_H_