File: encoded_image.h

package info (click to toggle)
chromium 139.0.7258.127-1
  • links: PTS, VCS
  • area: main
  • in suites:
  • size: 6,122,068 kB
  • sloc: cpp: 35,100,771; ansic: 7,163,530; javascript: 4,103,002; python: 1,436,920; asm: 946,517; xml: 746,709; pascal: 187,653; perl: 88,691; sh: 88,436; objc: 79,953; sql: 51,488; cs: 44,583; fortran: 24,137; makefile: 22,147; tcl: 15,277; php: 13,980; yacc: 8,984; ruby: 7,485; awk: 3,720; lisp: 3,096; lex: 1,327; ada: 727; jsp: 228; sed: 36
file content (303 lines) | stat: -rw-r--r-- 11,199 bytes parent folder | download | duplicates (3)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
/*
 *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#ifndef API_VIDEO_ENCODED_IMAGE_H_
#define API_VIDEO_ENCODED_IMAGE_H_

#include <stdint.h>

#include <cstddef>
#include <map>
#include <optional>
#include <utility>

#include "api/ref_count.h"
#include "api/rtp_packet_infos.h"
#include "api/scoped_refptr.h"
#include "api/units/timestamp.h"
#include "api/video/color_space.h"
#include "api/video/corruption_detection_filter_settings.h"
#include "api/video/video_codec_constants.h"
#include "api/video/video_content_type.h"
#include "api/video/video_frame_type.h"
#include "api/video/video_rotation.h"
#include "api/video/video_timing.h"
#include "rtc_base/buffer.h"
#include "rtc_base/checks.h"
#include "rtc_base/system/rtc_export.h"

namespace webrtc {

// Abstract interface for buffer storage. Intended to support buffers owned by
// external encoders with special release requirements, e.g, java encoders with
// releaseOutputBuffer.
class EncodedImageBufferInterface : public RefCountInterface {
 public:
  using value_type = uint8_t;

  virtual const uint8_t* data() const = 0;
  // TODO(bugs.webrtc.org/9378): Make interface essentially read-only, delete
  // this non-const data method.
  virtual uint8_t* data() = 0;
  virtual size_t size() const = 0;

  const uint8_t* begin() const { return data(); }
  const uint8_t* end() const { return data() + size(); }
};

// Basic implementation of EncodedImageBufferInterface.
class RTC_EXPORT EncodedImageBuffer : public EncodedImageBufferInterface {
 public:
  static scoped_refptr<EncodedImageBuffer> Create() { return Create(0); }
  static scoped_refptr<EncodedImageBuffer> Create(size_t size);
  static scoped_refptr<EncodedImageBuffer> Create(const uint8_t* data,
                                                  size_t size);
  static scoped_refptr<EncodedImageBuffer> Create(Buffer buffer);

  const uint8_t* data() const override;
  uint8_t* data() override;
  size_t size() const override;
  void Realloc(size_t t);

 protected:
  explicit EncodedImageBuffer(size_t size);
  EncodedImageBuffer(const uint8_t* data, size_t size);
  explicit EncodedImageBuffer(Buffer buffer);

  Buffer buffer_;
};

// TODO(bug.webrtc.org/9378): This is a legacy api class, which is slowly being
// cleaned up. Direct use of its members is strongly discouraged.
class RTC_EXPORT EncodedImage {
 public:
  EncodedImage();
  EncodedImage(EncodedImage&&);
  EncodedImage(const EncodedImage&);

  ~EncodedImage();

  EncodedImage& operator=(EncodedImage&&);
  EncodedImage& operator=(const EncodedImage&);

  // Frame capture time in RTP timestamp representation (90kHz).
  void SetRtpTimestamp(uint32_t timestamp) { timestamp_rtp_ = timestamp; }
  uint32_t RtpTimestamp() const { return timestamp_rtp_; }

  void SetEncodeTime(int64_t encode_start_ms, int64_t encode_finish_ms);

  // Frame capture time in local time.
  Timestamp CaptureTime() const;

  // Frame capture time in ntp epoch time, i.e. time since 1st Jan 1900
  int64_t NtpTimeMs() const { return ntp_time_ms_; }

  // Every simulcast layer (= encoding) has its own encoder and RTP stream.
  // There can be no dependencies between different simulcast layers.
  std::optional<int> SimulcastIndex() const { return simulcast_index_; }
  void SetSimulcastIndex(std::optional<int> simulcast_index) {
    RTC_DCHECK_GE(simulcast_index.value_or(0), 0);
    RTC_DCHECK_LT(simulcast_index.value_or(0), kMaxSimulcastStreams);
    simulcast_index_ = simulcast_index;
  }

  const std::optional<Timestamp>& PresentationTimestamp() const {
    return presentation_timestamp_;
  }
  void SetPresentationTimestamp(
      const std::optional<Timestamp>& presentation_timestamp) {
    presentation_timestamp_ = presentation_timestamp;
  }

  // Encoded images can have dependencies between spatial and/or temporal
  // layers, depending on the scalability mode used by the encoder. See diagrams
  // at https://w3c.github.io/webrtc-svc/#dependencydiagrams*.
  std::optional<int> SpatialIndex() const { return spatial_index_; }
  void SetSpatialIndex(std::optional<int> spatial_index) {
    RTC_DCHECK_GE(spatial_index.value_or(0), 0);
    RTC_DCHECK_LT(spatial_index.value_or(0), kMaxSpatialLayers);
    spatial_index_ = spatial_index;
  }

  std::optional<int> TemporalIndex() const { return temporal_index_; }
  void SetTemporalIndex(std::optional<int> temporal_index) {
    RTC_DCHECK_GE(temporal_index_.value_or(0), 0);
    RTC_DCHECK_LT(temporal_index_.value_or(0), kMaxTemporalStreams);
    temporal_index_ = temporal_index;
  }

  // These methods can be used to set/get size of subframe with spatial index
  // `spatial_index` on encoded frames that consist of multiple spatial layers.
  std::optional<size_t> SpatialLayerFrameSize(int spatial_index) const;
  void SetSpatialLayerFrameSize(int spatial_index, size_t size_bytes);

  const webrtc::ColorSpace* ColorSpace() const {
    return color_space_ ? &*color_space_ : nullptr;
  }
  void SetColorSpace(const std::optional<webrtc::ColorSpace>& color_space) {
    color_space_ = color_space;
  }

  std::optional<VideoPlayoutDelay> PlayoutDelay() const {
    return playout_delay_;
  }

  void SetPlayoutDelay(std::optional<VideoPlayoutDelay> playout_delay) {
    playout_delay_ = playout_delay;
  }

  // These methods along with the private member video_frame_tracking_id_ are
  // meant for media quality testing purpose only.
  std::optional<uint16_t> VideoFrameTrackingId() const {
    return video_frame_tracking_id_;
  }
  void SetVideoFrameTrackingId(std::optional<uint16_t> tracking_id) {
    video_frame_tracking_id_ = tracking_id;
  }

  const RtpPacketInfos& PacketInfos() const { return packet_infos_; }
  void SetPacketInfos(RtpPacketInfos packet_infos) {
    packet_infos_ = std::move(packet_infos);
  }

  bool RetransmissionAllowed() const { return retransmission_allowed_; }
  void SetRetransmissionAllowed(bool retransmission_allowed) {
    retransmission_allowed_ = retransmission_allowed;
  }

  size_t size() const { return size_; }
  void set_size(size_t new_size) {
    // Allow set_size(0) even if we have no buffer.
    RTC_DCHECK_LE(new_size, new_size == 0 ? 0 : capacity());
    size_ = new_size;
  }

  void SetEncodedData(scoped_refptr<EncodedImageBufferInterface> encoded_data) {
    encoded_data_ = encoded_data;
    size_ = encoded_data->size();
  }

  void ClearEncodedData() {
    encoded_data_ = nullptr;
    size_ = 0;
  }

  scoped_refptr<EncodedImageBufferInterface> GetEncodedData() const {
    return encoded_data_;
  }

  const uint8_t* data() const {
    return encoded_data_ ? encoded_data_->data() : nullptr;
  }

  const uint8_t* begin() const { return data(); }
  const uint8_t* end() const { return data() + size(); }

  // Returns whether the encoded image can be considered to be of target
  // quality.
  [[deprecated]] bool IsAtTargetQuality() const { return at_target_quality_; }

  // Sets that the encoded image can be considered to be of target quality to
  // true or false.
  [[deprecated]] void SetAtTargetQuality(bool at_target_quality) {
    at_target_quality_ = at_target_quality;
  }

  // Returns whether the frame that was encoded is a steady-state refresh frame
  // intended to improve the visual quality.
  bool IsSteadyStateRefreshFrame() const {
    return is_steady_state_refresh_frame_;
  }

  void SetIsSteadyStateRefreshFrame(bool refresh_frame) {
    is_steady_state_refresh_frame_ = refresh_frame;
  }

  VideoFrameType FrameType() const { return _frameType; }

  void SetFrameType(VideoFrameType frame_type) { _frameType = frame_type; }
  VideoContentType contentType() const { return content_type_; }
  VideoRotation rotation() const { return rotation_; }

  std::optional<CorruptionDetectionFilterSettings>
  corruption_detection_filter_settings() const {
    return corruption_detection_filter_settings_;
  }
  void set_corruption_detection_filter_settings(
      const CorruptionDetectionFilterSettings& settings) {
    corruption_detection_filter_settings_ = settings;
  }

  uint32_t _encodedWidth = 0;
  uint32_t _encodedHeight = 0;
  // NTP time of the capture time in local timebase in milliseconds.
  // TODO(minyue): make this member private.
  int64_t ntp_time_ms_ = 0;
  int64_t capture_time_ms_ = 0;
  VideoFrameType _frameType = VideoFrameType::kVideoFrameDelta;
  VideoRotation rotation_ = kVideoRotation_0;
  VideoContentType content_type_ = VideoContentType::UNSPECIFIED;
  int qp_ = -1;  // Quantizer value.

  struct Timing {
    uint8_t flags = VideoSendTiming::kInvalid;
    int64_t encode_start_ms = 0;
    int64_t encode_finish_ms = 0;
    int64_t packetization_finish_ms = 0;
    int64_t pacer_exit_ms = 0;
    int64_t network_timestamp_ms = 0;
    int64_t network2_timestamp_ms = 0;
    int64_t receive_start_ms = 0;
    int64_t receive_finish_ms = 0;
  } timing_;
  EncodedImage::Timing video_timing() const { return timing_; }
  EncodedImage::Timing* video_timing_mutable() { return &timing_; }

 private:
  size_t capacity() const { return encoded_data_ ? encoded_data_->size() : 0; }

  // When set, indicates that all future frames will be constrained with those
  // limits until the application indicates a change again.
  std::optional<VideoPlayoutDelay> playout_delay_;

  scoped_refptr<EncodedImageBufferInterface> encoded_data_;
  size_t size_ = 0;  // Size of encoded frame data.
  uint32_t timestamp_rtp_ = 0;
  std::optional<int> simulcast_index_;
  std::optional<Timestamp> presentation_timestamp_;
  std::optional<int> spatial_index_;
  std::optional<int> temporal_index_;
  std::map<int, size_t> spatial_layer_frame_size_bytes_;
  std::optional<webrtc::ColorSpace> color_space_;
  // This field is meant for media quality testing purpose only. When enabled it
  // carries the VideoFrame id field from the sender to the receiver.
  std::optional<uint16_t> video_frame_tracking_id_;
  // Information about packets used to assemble this video frame. This is needed
  // by `SourceTracker` when the frame is delivered to the RTCRtpReceiver's
  // MediaStreamTrack, in order to implement getContributingSources(). See:
  // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources
  RtpPacketInfos packet_infos_;
  bool retransmission_allowed_ = true;
  // True if the encoded image can be considered to be of target quality.
  bool at_target_quality_ = false;
  // True if the frame that was encoded is a steady-state refresh frame intended
  // to improve the visual quality.
  bool is_steady_state_refresh_frame_ = false;

  // Filter settings for corruption detection suggested by the encoder
  // implementation, if any. Otherwise generic per-codec-type settings will be
  // used.
  std::optional<CorruptionDetectionFilterSettings>
      corruption_detection_filter_settings_;
};

}  // namespace webrtc

#endif  // API_VIDEO_ENCODED_IMAGE_H_