1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
|
// Copyright 2021 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_WEBGPU_GPU_EXTERNAL_TEXTURE_H_
#define THIRD_PARTY_BLINK_RENDERER_MODULES_WEBGPU_GPU_EXTERNAL_TEXTURE_H_
#include <atomic>
#include "base/task/single_thread_task_runner.h"
#include "media/base/video_frame.h"
#include "third_party/blink/renderer/modules/webgpu/dawn_object.h"
#include "third_party/blink/renderer/platform/bindings/script_wrappable.h"
#include "third_party/blink/renderer/platform/heap/collection_support/heap_hash_map.h"
#include "third_party/blink/renderer/platform/heap/collection_support/heap_hash_set.h"
#include "third_party/blink/renderer/platform/heap/garbage_collected.h"
#include "third_party/blink/renderer/platform/wtf/ref_counted.h"
namespace media {
class PaintCanvasVideoRenderer;
} // namespace media
namespace blink {
class ExceptionState;
class GPUExternalTexture;
class GPUExternalTextureDescriptor;
class WebGPUMailboxTexture;
class HTMLVideoElement;
class VideoFrame;
// GPUExternalTexture uses auto expiry mechanism
// (https://www.w3.org/TR/webgpu/#-automatic-expiry-task-source). The
// mechanism requires webgpu to expire GPUExternalTexture when current task
// scope finished by posting expiration task. The expired GPUExternalTexture
// is invalid to submit and needs to call importExternalTexture() to get the
// refreshed GPUExternalTexture object. In implementation side,
// importExternalTexture() also wraps GPUExternalTexture with underly video
// frames. It is possible that multiple importExternalTexture() call with the
// same source tries to wrap the same underly video frames. So a cache system
// has been integrated to avoid re-creating the GPUExternalTexture again and
// again with the same video frame. So importExternalTexture() tries to do:
// - Search cache to see any hit. if not, create a new GPUExternalTexture and
// insert it into cache.
// - Refresh the external texture to un-expire it.
// - Post a task to expire this external texture after finishing current task.
// More details refers to
// https://www.w3.org/TR/webgpu/#external-texture-creation
class ExternalTextureCache : public GarbageCollected<ExternalTextureCache> {
public:
explicit ExternalTextureCache(GPUDevice* device);
ExternalTextureCache(const ExternalTextureCache&) = delete;
ExternalTextureCache& operator=(const ExternalTextureCache&) = delete;
// Implement importExternalTexture() auto expiry mechanism.
GPUExternalTexture* Import(const GPUExternalTextureDescriptor* descriptor,
ExceptionState& exception_state);
// Destroy all cached GPUExternalTexture and clear all lists.
void Destroy();
void Add(HTMLVideoElement* video, GPUExternalTexture* external_texture);
void Remove(HTMLVideoElement* video);
void Add(VideoFrame* frame, GPUExternalTexture* external_texture);
void Remove(VideoFrame* frame);
void ReferenceUntilGPUIsFinished(
scoped_refptr<WebGPUMailboxTexture> mailbox_texture);
void Trace(Visitor* visitor) const;
GPUDevice* device() const;
private:
void ExpireAtEndOfTask(GPUExternalTexture* external_texture);
void ExpireTask();
// Keep a list of all active GPUExternalTexture. Eagerly destroy them
// when the device is destroyed (via .destroy) to free the memory.
HeapHashMap<WeakMember<HTMLVideoElement>, WeakMember<GPUExternalTexture>>
from_html_video_element_;
HeapHashMap<WeakMember<VideoFrame>, WeakMember<GPUExternalTexture>>
from_video_frame_;
bool expire_task_scheduled_ = false;
HeapHashSet<Member<GPUExternalTexture>> expire_set_;
Member<GPUDevice> device_;
};
class GPUExternalTexture : public DawnObject<wgpu::ExternalTexture> {
DEFINE_WRAPPERTYPEINFO();
public:
static GPUExternalTexture* CreateExpired(
ExternalTextureCache* cache,
const GPUExternalTextureDescriptor* webgpu_desc,
ExceptionState& exception_state);
static GPUExternalTexture* FromHTMLVideoElement(
ExternalTextureCache* cache,
HTMLVideoElement* video,
const GPUExternalTextureDescriptor* webgpu_desc,
ExceptionState& exception_state);
static GPUExternalTexture* FromVideoFrame(
ExternalTextureCache* cache,
VideoFrame* frame,
const GPUExternalTextureDescriptor* webgpu_desc,
ExceptionState& exception_state);
explicit GPUExternalTexture(
ExternalTextureCache* cache,
wgpu::ExternalTexture external_texture,
scoped_refptr<WebGPUMailboxTexture> mailbox_texture,
bool is_zero_copy,
bool read_lock_fences_enabled,
std::optional<media::VideoFrame::ID> media_video_frame_unique_id,
const String& label);
GPUExternalTexture(const GPUExternalTexture&) = delete;
GPUExternalTexture& operator=(const GPUExternalTexture&) = delete;
// gpu_external_texture.idl {{{
bool isZeroCopy() const;
// }}} End of WebIDL binding implementation.
bool IsReadLockFenceEnabled() const;
void Destroy();
void Expire();
void Refresh();
void SetVideo(HTMLVideoElement* video);
// Returns true iff the video frame is still available
bool ListenToVideoFrame(VideoFrame* frame);
// Check whether current VideoFrame is outdated from HTMLVideoElement. Pure
// video playback might not trigger any script animation work. Check video
// frame states in import to ensure cache refresh.
bool NeedsToUpdate();
// GPUExternalTexture from VideoFrame expires when VideoFrame is closed. Note
// that all back resources destroyed needs to happen on the thread that
// GPUExternalTexture is created.
// In multithread situation, the callback should change the state of external
// texture to State::Expired and post a task to issue the destroy.
void OnVideoFrameClosed();
void Trace(Visitor* visitor) const override;
private:
// [1] [2]
// Creation -> [Active] --> [Expired] --> [Destroyed]
// ^ |
// |-------------
// [3]
//
// [1] Happens when the current task finishes: the GPUExternalTexture cannot
// be used util it is refreshed [2] Happens when the source changes frames,
// the texture can no longer be refreshed. [3] Happens when the texture is
// refreshed by being re-imported.
enum class Status { Active, Expired, Destroyed };
static GPUExternalTexture* CreateImpl(
ExternalTextureCache* cache,
const GPUExternalTextureDescriptor* webgpu_desc,
scoped_refptr<media::VideoFrame> media_video_frame,
media::PaintCanvasVideoRenderer* video_renderer,
std::optional<media::VideoFrame::ID> media_video_frame_unique_id,
ExceptionState& exception_state);
void SetLabelImpl(const String& value) override {
std::string utf8_label = value.Utf8();
GetHandle().SetLabel(utf8_label.c_str());
}
bool IsCurrentFrameFromHTMLVideoElementValid();
// This is the function to push a task to destroy the external texture when
// the imported video frame in GPUDevice cache is outdated. The function is
// used as callback function and be registered to the imported
// Blink::VideoFrame or HTMLVideoElement.
void OnSourceInvalidated();
// GPUDevice holds cache for GPUExternalTextures to handling import same
// frame multiple time cases.
void RemoveFromCache();
bool IsActive() const;
bool IsExpired() const;
bool IsDestroyed() const;
scoped_refptr<WebGPUMailboxTexture> mailbox_texture_;
bool is_zero_copy_ = false;
bool remove_from_cache_task_scheduled_ = false;
// read_lock_fences_enabled_ comes from media::VideoFrame metadata.
// VideoFrame set this metadata as a hint to ensure all previous gpu
// execution complete before returning video frame to producer.
bool read_lock_fences_enabled_ = false;
std::optional<media::VideoFrame::ID> media_video_frame_unique_id_;
WeakMember<HTMLVideoElement> video_;
WeakMember<VideoFrame> frame_;
WeakMember<ExternalTextureCache> cache_;
scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
std::atomic<Status> status_ = Status::Active;
};
} // namespace blink
#endif // THIRD_PARTY_BLINK_RENDERER_MODULES_WEBGPU_GPU_EXTERNAL_TEXTURE_H_
|
