1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
|
// Copyright 2019 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef CONTENT_RENDERER_ACCESSIBILITY_AX_IMAGE_ANNOTATOR_H_
#define CONTENT_RENDERER_ACCESSIBILITY_AX_IMAGE_ANNOTATOR_H_
#include <string>
#include <unordered_map>
#include "base/functional/bind.h"
#include "base/memory/raw_ptr.h"
#include "base/memory/weak_ptr.h"
#include "base/observer_list_types.h"
#include "content/common/content_export.h"
#include "content/renderer/accessibility/render_accessibility_impl.h"
#include "mojo/public/cpp/bindings/pending_remote.h"
#include "mojo/public/cpp/bindings/remote.h"
#include "services/image_annotation/public/cpp/image_processor.h"
#include "services/image_annotation/public/mojom/image_annotation.mojom-forward.h"
#include "third_party/abseil-cpp/absl/types/optional.h"
#include "third_party/skia/include/core/SkBitmap.h"
#include "ui/accessibility/ax_enums.mojom.h"
namespace blink {
class WebAXObject;
} // namespace blink
namespace content {
class ContentClient;
// This class gets notified that certain images have been added, removed or
// updated on a page. This class is then responsible for retrieving the
// automatic label for all images and notifying the RenderAccessibility that
// owns it to update the relevant image annotations.
class CONTENT_EXPORT AXImageAnnotator : public base::CheckedObserver {
public:
AXImageAnnotator(
RenderAccessibilityImpl* const render_accessibility,
mojo::PendingRemote<image_annotation::mojom::Annotator> annotator);
AXImageAnnotator(const AXImageAnnotator&) = delete;
AXImageAnnotator& operator=(const AXImageAnnotator&) = delete;
~AXImageAnnotator() override;
void Destroy();
std::string GetImageAnnotation(blink::WebAXObject& image) const;
ax::mojom::ImageAnnotationStatus GetImageAnnotationStatus(
blink::WebAXObject& image) const;
bool HasAnnotationInCache(blink::WebAXObject& image) const;
bool HasImageInCache(const blink::WebAXObject& image) const;
void OnImageAdded(blink::WebAXObject& image);
void OnImageUpdated(blink::WebAXObject& image);
void OnImageRemoved(blink::WebAXObject& image);
static int GetLengthAfterRemovingStopwords(const std::string& image_name);
static bool ImageNameHasMostlyStopwords(const std::string& image_name);
private:
// Keeps track of the image data and the automatic annotation for each image.
class ImageInfo final {
public:
ImageInfo(const blink::WebAXObject& image);
virtual ~ImageInfo();
mojo::PendingRemote<image_annotation::mojom::ImageProcessor>
GetImageProcessor();
bool HasAnnotation() const;
ax::mojom::ImageAnnotationStatus status() const { return status_; }
void set_status(ax::mojom::ImageAnnotationStatus status) {
DCHECK_NE(status, ax::mojom::ImageAnnotationStatus::kNone);
status_ = status;
}
std::string annotation() const {
return annotation_.value_or("");
}
void set_annotation(std::string annotation) { annotation_ = annotation; }
private:
image_annotation::ImageProcessor image_processor_;
ax::mojom::ImageAnnotationStatus status_;
absl::optional<std::string> annotation_;
};
// Retrieves the image data from the renderer.
static SkBitmap GetImageData(const blink::WebAXObject& image);
// Used by tests to override the content client.
virtual ContentClient* GetContentClient() const;
// Given a WebImage, it uses the URL of the main document and the src
// attribute of the image, to generate a unique identifier for the image that
// could be provided to the image annotation service.
//
// This method is virtual to allow overriding it from tests.
virtual std::string GenerateImageSourceId(
const blink::WebAXObject& image) const;
// Removes the automatic image annotations from all images.
void MarkAllImagesDirty();
// Marks a node in the accessibility tree dirty when an image annotation
// changes. Also marks dirty a link or document that immediately contains
// an image.
void MarkDirty(const blink::WebAXObject& image) const;
// Gets called when an image gets annotated by the image annotation service.
void OnImageAnnotated(const blink::WebAXObject& image,
image_annotation::mojom::AnnotateImageResultPtr result);
// Only for local logging when running with --v=1.
std::string GetDocumentUrl() const;
// Weak, owns us.
const raw_ptr<RenderAccessibilityImpl, ExperimentalRenderer>
render_accessibility_;
// A pointer to the automatic image annotation service.
mojo::Remote<image_annotation::mojom::Annotator> annotator_;
// Keeps track of the image data and the automatic annotations for each image.
//
// The key is retrieved using WebAXObject::AxID().
std::unordered_map<int, ImageInfo> image_annotations_;
// This member needs to be last because it should destructed first.
base::WeakPtrFactory<AXImageAnnotator> weak_factory_{this};
};
} // namespace content
#endif // CONTENT_RENDERER_ACCESSIBILITY_AX_IMAGE_ANNOTATOR_H_
|
