File: ax_tree_distiller.h

package info (click to toggle)
chromium 139.0.7258.127-1
  • links: PTS, VCS
  • area: main
  • in suites:
  • size: 6,122,068 kB
  • sloc: cpp: 35,100,771; ansic: 7,163,530; javascript: 4,103,002; python: 1,436,920; asm: 946,517; xml: 746,709; pascal: 187,653; perl: 88,691; sh: 88,436; objc: 79,953; sql: 51,488; cs: 44,583; fortran: 24,137; makefile: 22,147; tcl: 15,277; php: 13,980; yacc: 8,984; ruby: 7,485; awk: 3,720; lisp: 3,096; lex: 1,327; ada: 727; jsp: 228; sed: 36
file content (139 lines) | stat: -rw-r--r-- 5,551 bytes parent folder | download | duplicates (6)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
// Copyright 2022 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifndef CHROME_RENDERER_ACCESSIBILITY_AX_TREE_DISTILLER_H_
#define CHROME_RENDERER_ACCESSIBILITY_AX_TREE_DISTILLER_H_

#include <memory>
#include <vector>

#include "base/functional/callback.h"
#include "base/memory/weak_ptr.h"
#include "base/scoped_observation.h"
#include "base/time/time.h"
#include "content/public/renderer/render_frame_observer.h"
#include "mojo/public/cpp/bindings/remote.h"
#include "services/metrics/public/cpp/ukm_source_id.h"
#include "services/screen_ai/public/mojom/screen_ai_service.mojom.h"
#include "ui/accessibility/ax_node_id_forward.h"
#include "ui/accessibility/ax_tree_id.h"
#include "ui/accessibility/ax_tree_update_forward.h"

namespace content {
class RenderFrame;
}

namespace ui {
class AXTree;
}

namespace ukm {
class MojoUkmRecorder;
}

///////////////////////////////////////////////////////////////////////////////
// AXTreeDistiller
//
//  A class that distills an AXTreeUpdate. The main API is
//  AXTreeDistiller::Distill(), which kicks off the distillation. Once a
//  distilled AXTree is ready, calls a callback.
//  If ChromeScreenAI library is available, the distillation is performed by the
//  Screen2x ML model in the utility process. Otherwise, distillation is done
//  using rules defined in this file.
//
class AXTreeDistiller : public content::RenderFrameObserver {
  using OnAXTreeDistilledCallback = base::RepeatingCallback<void(
      const ui::AXTreeID& tree_id,
      const std::vector<ui::AXNodeID>& content_node_ids)>;

 public:
  explicit AXTreeDistiller(
      content::RenderFrame* render_frame,
      OnAXTreeDistilledCallback on_ax_tree_distilled_callback);
  ~AXTreeDistiller() override;
  AXTreeDistiller(const AXTreeDistiller&) = delete;
  AXTreeDistiller& operator=(const AXTreeDistiller&) = delete;

  // Distills the AXTree. |tree| and |snapshot| are duplicates of each other;
  // the algorithm requires the data in |tree| form while Screen2x requires it
  // in |snapshot| form.
  // When ChromeScreenAI library is available, this operation is done in the
  // utility process by Screen2x. Otherwise, it is done by a rules-based
  // algorithm in this process.
  virtual void Distill(const ui::AXTree& tree,
                       const ui::AXTreeUpdate& snapshot,
                       const ukm::SourceId ukm_source_id);

  void ScreenAIServiceReady();

  // content::RenderFrameObserver:
  void OnDestruct() override {}

 private:
  // Distills the AXTree via a rules-based algorithm. Results are added to
  // |content_node_ids|.
  void DistillViaAlgorithm(const ui::AXTree& tree,
                           const ukm::SourceId ukm_source_id,
                           std::vector<ui::AXNodeID>* content_node_ids);

  void RecordRulesMetrics(const ukm::SourceId ukm_source_id,
                          base::TimeDelta elapsed_time,
                          bool success);
  void RecordScreen2xMetrics(const ukm::SourceId ukm_source_id,
                             base::TimeDelta elapsed_time,
                             bool success);

  // Passes |snapshot| to the Screen2x ML model, which identifes the main
  // content nodes and calls |ProcessScreen2xResult()| on completion.
  // |content_node_ids_algorithm| are the content nodes identified by the
  // algorithm. They are passed along to the screen2x callback.
  // |merged_start_time| is the time when Distill started and is used for
  // RecordMergedMetrics.
  void DistillViaScreen2x(
      const ui::AXTree& tree,
      const ui::AXTreeUpdate& snapshot,
      const ukm::SourceId ukm_source_id,
      base::TimeTicks merged_start_time,
      std::vector<ui::AXNodeID>* content_node_ids_algorithm);

  // Called by the Screen2x service from the utility process. Merges the result
  // from the algorithm with the result from Screen2x and passes the merged
  // vector to the callback. |screen2x_start_time| is the time when
  // DistillViaScreen2x started and |merged_start_time| is the time when
  // Distill started and is used for RecordScreen2xMetrics.
  void ProcessScreen2xResult(
      const ui::AXTreeID& tree_id,
      const ukm::SourceId ukm_source_id,
      base::TimeTicks screen2x_start_time,
      base::TimeTicks merged_start_time,
      std::vector<ui::AXNodeID> content_node_ids_algorithm,
      const std::vector<ui::AXNodeID>& content_node_ids_screen2x);

  // Called when the main content extractor is disconnected. Runs the callback
  // with an empty list of content node IDs.
  void OnMainContentExtractorDisconnected();

  // Record time it takes for the merged algorithm (distillation via algorithm
  // and via Screen2x) to run.
  void RecordMergedMetrics(const ukm::SourceId ukm_source_id,
                           base::TimeDelta elapsed_time,
                           bool success);

  // TODO(crbug.com/40802192): Ensure this is called even if ScreenAIService is
  // disconnected.
  OnAXTreeDistilledCallback on_ax_tree_distilled_callback_;

  std::unique_ptr<ukm::MojoUkmRecorder> ukm_recorder_;

  // ScreenAI service is successfully initialized.
  bool screen_ai_service_ready_ = false;

  // The remote of the Screen2x main content extractor. The receiver lives in
  // the utility process.
  mojo::Remote<screen_ai::mojom::Screen2xMainContentExtractor>
      main_content_extractor_;
  base::WeakPtrFactory<AXTreeDistiller> weak_ptr_factory_{this};
};

#endif  // CHROME_RENDERER_ACCESSIBILITY_AX_TREE_DISTILLER_H_