File: facegaze_test_utils.h

package info (click to toggle)
chromium 139.0.7258.127-1
  • links: PTS, VCS
  • area: main
  • in suites:
  • size: 6,122,068 kB
  • sloc: cpp: 35,100,771; ansic: 7,163,530; javascript: 4,103,002; python: 1,436,920; asm: 946,517; xml: 746,709; pascal: 187,653; perl: 88,691; sh: 88,436; objc: 79,953; sql: 51,488; cs: 44,583; fortran: 24,137; makefile: 22,147; tcl: 15,277; php: 13,980; yacc: 8,984; ruby: 7,485; awk: 3,720; lisp: 3,096; lex: 1,327; ada: 727; jsp: 228; sed: 36
file content (306 lines) | stat: -rw-r--r-- 9,697 bytes parent folder | download | duplicates (5)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
// Copyright 2024 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifndef CHROME_BROWSER_ASH_ACCESSIBILITY_FACEGAZE_TEST_UTILS_H_
#define CHROME_BROWSER_ASH_ACCESSIBILITY_FACEGAZE_TEST_UTILS_H_

#include <memory>
#include <optional>
#include <string>
#include <utility>

#include "base/containers/flat_map.h"
#include "base/values.h"
#include "ui/events/test/event_generator.h"
#include "ui/gfx/geometry/point_f.h"

namespace gfx {
class Point;
}  // namespace gfx

namespace ash {

// A class that can be used to exercise FaceGaze in browsertests.
class FaceGazeTestUtils {
 public:
  // The facial gestures that are supported by FaceGaze. Ensure this enum stays
  // in sync with the source of truth in
  // ash/webui/common/resources/accessibility/facial_gestures.ts.
  enum class FaceGazeGesture {
    BROW_INNER_UP,
    BROWS_DOWN,
    EYE_SQUINT_LEFT,
    EYE_SQUINT_RIGHT,
    EYES_BLINK,
    EYES_LOOK_DOWN,
    EYES_LOOK_LEFT,
    EYES_LOOK_RIGHT,
    EYES_LOOK_UP,
    JAW_LEFT,
    JAW_OPEN,
    JAW_RIGHT,
    MOUTH_FUNNEL,
    MOUTH_LEFT,
    MOUTH_PUCKER,
    MOUTH_RIGHT,
    MOUTH_SMILE,
    MOUTH_UPPER_UP,
  };

  static std::string ToString(const FaceGazeGesture& gesture);

  // Macros used by accessibility features on ChromeOS.
  // Ensure this enum stays in sync with the source of truth in
  // ash/webui/common/resources/accessibility/macro_names.ts.
  enum MacroName {
    UNSPECIFIED = 0,
    INPUT_TEXT_VIEW = 1,
    DELETE_PREV_CHAR = 2,
    NAV_PREV_CHAR = 3,
    NAV_NEXT_CHAR = 4,
    NAV_PREV_LINE = 5,
    NAV_NEXT_LINE = 6,
    COPY_SELECTED_TEXT = 7,
    PASTE_TEXT = 8,
    CUT_SELECTED_TEXT = 9,
    UNDO_TEXT_EDIT = 10,
    REDO_ACTION = 11,
    SELECT_ALL_TEXT = 12,
    UNSELECT_TEXT = 13,
    LIST_COMMANDS = 14,
    NEW_LINE = 15,
    TOGGLE_DICTATION = 16,
    DELETE_PREV_WORD = 17,
    DELETE_PREV_SENT = 18,
    NAV_NEXT_WORD = 19,
    NAV_PREV_WORD = 20,
    SMART_DELETE_PHRASE = 21,
    SMART_REPLACE_PHRASE = 22,
    SMART_INSERT_BEFORE = 23,
    SMART_SELECT_BTWN_INCL = 24,
    NAV_NEXT_SENT = 25,
    NAV_PREV_SENT = 26,
    DELETE_ALL_TEXT = 27,
    NAV_START_TEXT = 28,
    NAV_END_TEXT = 29,
    SELECT_PREV_WORD = 30,
    SELECT_NEXT_WORD = 31,
    SELECT_NEXT_CHAR = 32,
    SELECT_PREV_CHAR = 33,
    REPEAT = 34,
    MOUSE_CLICK_LEFT = 35,
    MOUSE_CLICK_RIGHT = 36,
    RESET_CURSOR = 37,
    KEY_PRESS_SPACE = 38,
    KEY_PRESS_LEFT = 39,
    KEY_PRESS_RIGHT = 40,
    KEY_PRESS_UP = 41,
    KEY_PRESS_DOWN = 42,
    MOUSE_LONG_CLICK_LEFT = 45,
    TOGGLE_FACEGAZE = 46,
    OPEN_FACEGAZE_SETTINGS = 47,
    TOGGLE_VIRTUAL_KEYBOARD = 48,
    MOUSE_CLICK_LEFT_DOUBLE = 49,
    TOGGLE_SCROLL_MODE = 50,
    CUSTOM_KEY_COMBINATION = 51,
    KEY_PRESS_SCREENSHOT = 52,
    MOUSE_CLICK_LEFT_TRIPLE = 53,
    TOGGLE_PRECISION_CLICK = 54,
  };

  // Facial gestures recognized by Mediapipe. Ensure this enum stays in sync
  // with the source of truth in chrome/browser/resources/chromeos/\
  // accessibility/accessibility_common/facegaze/gesture_detector.ts.
  enum class MediapipeGesture {
    BROW_DOWN_LEFT,
    BROW_DOWN_RIGHT,
    BROW_INNER_UP,
    EYE_BLINK_LEFT,
    EYE_BLINK_RIGHT,
    EYE_LOOK_DOWN_LEFT,
    EYE_LOOK_DOWN_RIGHT,
    EYE_LOOK_IN_LEFT,
    EYE_LOOK_IN_RIGHT,
    EYE_LOOK_OUT_LEFT,
    EYE_LOOK_OUT_RIGHT,
    EYE_LOOK_UP_LEFT,
    EYE_LOOK_UP_RIGHT,
    EYE_SQUINT_LEFT,
    EYE_SQUINT_RIGHT,
    JAW_LEFT,
    JAW_OPEN,
    JAW_RIGHT,
    MOUTH_FUNNEL,
    MOUTH_LEFT,
    MOUTH_PUCKER,
    MOUTH_RIGHT,
    MOUTH_SMILE_LEFT,
    MOUTH_SMILE_RIGHT,
    MOUTH_UPPER_UP_LEFT,
    MOUTH_UPPER_UP_RIGHT,
  };

  static std::string ToString(const MediapipeGesture& gesture);

  // A struct that holds cursor speed values.
  struct CursorSpeeds {
    int up;
    int down;
    int left;
    int right;
  };

  // A class that helps initialize FaceGaze with a configuration.
  class Config {
   public:
    Config();
    ~Config();
    Config(const Config&) = delete;
    Config& operator=(const Config&) = delete;

    // Returns a Config that sets required properties to default values.
    Config& Default();
    Config& WithForeheadLocation(const gfx::PointF& location);
    Config& WithCursorLocation(const gfx::Point& location);
    Config& WithBufferSize(int size);
    Config& WithCursorAcceleration(bool acceleration);
    Config& WithDialogAccepted(bool accepted);
    Config& WithBindings(
        const base::flat_map<FaceGazeGesture, MacroName>& gestures_to_macros,
        const base::flat_map<FaceGazeGesture, int>& gesture_confidences);
    Config& WithCursorSpeeds(const CursorSpeeds& speeds);
    Config& WithGestureRepeatDelayMs(int delay);
    Config& WithLandmarkWeights(bool use_weights);
    Config& WithVelocityThreshold(bool use_threshold);

    const gfx::PointF& forehead_location() const { return forehead_location_; }
    const gfx::Point& cursor_location() const { return cursor_location_; }
    int buffer_size() const { return buffer_size_; }
    bool use_cursor_acceleration() const { return use_cursor_acceleration_; }
    bool use_landmark_weights() const { return use_landmark_weights_; }
    bool use_velocity_threshold() const { return use_velocity_threshold_; }
    bool dialog_accepted() const { return dialog_accepted_; }
    bool use_gesture_duration() const { return use_gesture_duration_; }
    const std::optional<base::flat_map<FaceGazeGesture, MacroName>>&
    gestures_to_macros() const {
      return gestures_to_macros_;
    }
    const std::optional<base::flat_map<FaceGazeGesture, int>>&
    gesture_confidences() const {
      return gesture_confidences_;
    }
    const std::optional<CursorSpeeds>& cursor_speeds() const {
      return cursor_speeds_;
    }
    std::optional<int> gesture_repeat_delay_ms() const {
      return gesture_repeat_delay_ms_;
    }

   private:
    // Required properties.
    gfx::PointF forehead_location_;
    gfx::Point cursor_location_;
    int buffer_size_;
    bool use_cursor_acceleration_;
    bool use_landmark_weights_;
    bool use_velocity_threshold_;
    bool dialog_accepted_;
    bool use_gesture_duration_;

    // Optional properties.
    std::optional<base::flat_map<FaceGazeGesture, MacroName>>
        gestures_to_macros_;
    std::optional<base::flat_map<FaceGazeGesture, int>> gesture_confidences_;
    std::optional<CursorSpeeds> cursor_speeds_;
    std::optional<int> gesture_repeat_delay_ms_;
  };

  // A class that represents a fake FaceLandmarkerResult.
  class MockFaceLandmarkerResult {
   public:
    MockFaceLandmarkerResult();
    ~MockFaceLandmarkerResult();
    MockFaceLandmarkerResult(const MockFaceLandmarkerResult&) = delete;
    MockFaceLandmarkerResult& operator=(const MockFaceLandmarkerResult&) =
        delete;

    MockFaceLandmarkerResult& WithNormalizedForeheadLocation(
        const std::pair<double, double>& location);
    MockFaceLandmarkerResult& WithGesture(const MediapipeGesture& gesture,
                                          int confidence);

    MockFaceLandmarkerResult& WithLatency(int latency);

    const base::Value::Dict& forehead_location() const {
      return forehead_location_;
    }
    const base::Value::List& recognized_gestures() const {
      return recognized_gestures_;
    }

    const std::optional<int>& latency() const { return latency_; }

   private:
    std::optional<int> latency_;
    base::Value::Dict forehead_location_;
    base::Value::List recognized_gestures_;
  };

  FaceGazeTestUtils();
  ~FaceGazeTestUtils();
  FaceGazeTestUtils(const FaceGazeTestUtils&) = delete;
  FaceGazeTestUtils& operator=(const FaceGazeTestUtils&) = delete;

  // Enables, sets up, and configures FaceGaze with the given configuration.
  void EnableFaceGaze(const Config& config);
  // Waits for the cursor location to propagate to the FaceGaze MouseController.
  void WaitForCursorPosition(const gfx::Point& location);
  // Forces FaceGaze to process `result`, since tests don't have access to real
  // camera data.
  void ProcessFaceLandmarkerResult(const MockFaceLandmarkerResult& result);
  // The MouseController updates the cursor location at a set interval. To
  // increase test stability, the interval is canceled in tests, and must be
  // triggered manually using this method.
  void TriggerMouseControllerInterval();

  void MoveMouseTo(const gfx::Point& location);
  void AssertCursorAt(const gfx::Point& location);

  void AssertScrollMode(bool active);

  void WaitForFaceLandmarker();

 private:
  void ExecuteAccessibilityCommonScript(const std::string& script);

  // Setup-related methods.
  void SetUpMediapipeDir(const char* mediapipe_dir);
  void WaitForJSReady();
  void SetUpJSTestSupport(const char* test_support_dir);
  void CancelMouseControllerInterval();
  void ConfigureFaceGaze(const Config& config);

  // Preference-related methods.
  void SetCursorSpeeds(const CursorSpeeds& speeds);
  void SetBufferSize(int size);
  void SetCursorAcceleration(bool use_acceleration);
  void SetLandmarkWeights(bool use_weights);
  void SetVelocityThreshold(bool use_threshold);
  void SetGesturesToMacros(
      const base::flat_map<FaceGazeGesture, MacroName>& gestures_to_macros);
  void SetGestureConfidences(
      const base::flat_map<FaceGazeGesture, int>& gesture_confidences);

  // Sets the gesture repeat delay threshold.
  void SetGestureRepeatDelayMs(int delay);

  // Sets whether a gesture duration should be required to recognize a gesture.
  void SetGestureDuration(bool use_duration);

  std::unique_ptr<ui::test::EventGenerator> event_generator_;
};

}  // namespace ash

#endif  // CHROME_BROWSER_ASH_ACCESSIBILITY_FACEGAZE_TEST_UTILS_H_