1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234
|
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
package org.webrtc;
import android.graphics.Matrix;
import android.opengl.GLES11Ext;
import android.opengl.GLES20;
import androidx.annotation.Nullable;
import java.nio.ByteBuffer;
/**
* Java version of webrtc::VideoFrame and webrtc::VideoFrameBuffer. A difference from the C++
* version is that no explicit tag is used, and clients are expected to use 'instanceof' to find the
* right subclass of the buffer. This allows clients to create custom VideoFrame.Buffer in
* arbitrary format in their custom VideoSources, and then cast it back to the correct subclass in
* their custom VideoSinks. All implementations must also implement the toI420() function,
* converting from the underlying representation if necessary. I420 is the most widely accepted
* format and serves as a fallback for video sinks that can only handle I420, e.g. the internal
* WebRTC software encoders.
*/
public class VideoFrame implements RefCounted {
/**
* Implements image storage medium. Might be for example an OpenGL texture or a memory region
* containing I420-data.
*
* <p>Reference counting is needed since a video buffer can be shared between multiple VideoSinks,
* and the buffer needs to be returned to the VideoSource as soon as all references are gone.
*/
public interface Buffer extends RefCounted {
/**
* Representation of the underlying buffer. Currently, only NATIVE and I420 are supported.
*/
@CalledByNative("Buffer")
@VideoFrameBufferType
default int getBufferType() {
return VideoFrameBufferType.NATIVE;
}
/**
* Resolution of the buffer in pixels.
*/
@CalledByNative("Buffer") int getWidth();
@CalledByNative("Buffer") int getHeight();
/**
* Returns a memory-backed frame in I420 format. If the pixel data is in another format, a
* conversion will take place. All implementations must provide a fallback to I420 for
* compatibility with e.g. the internal WebRTC software encoders.
*
* <p> Conversion may fail, for example if reading the pixel data from a texture fails. If the
* conversion fails, null is returned.
*/
@Nullable @CalledByNative("Buffer") I420Buffer toI420();
@Override @CalledByNative("Buffer") void retain();
@Override @CalledByNative("Buffer") void release();
/**
* Crops a region defined by `cropx`, `cropY`, `cropWidth` and `cropHeight`. Scales it to size
* `scaleWidth` x `scaleHeight`.
*/
@CalledByNative("Buffer")
Buffer cropAndScale(
int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight);
}
/**
* Interface for I420 buffers.
*/
public interface I420Buffer extends Buffer {
@Override
default int getBufferType() {
return VideoFrameBufferType.I420;
}
/**
* Returns a direct ByteBuffer containing Y-plane data. The buffer capacity is at least
* getStrideY() * getHeight() bytes. The position of the returned buffer is ignored and must
* be 0. Callers may mutate the ByteBuffer (eg. through relative-read operations), so
* implementations must return a new ByteBuffer or slice for each call.
*/
@CalledByNative("I420Buffer") ByteBuffer getDataY();
/**
* Returns a direct ByteBuffer containing U-plane data. The buffer capacity is at least
* getStrideU() * ((getHeight() + 1) / 2) bytes. The position of the returned buffer is ignored
* and must be 0. Callers may mutate the ByteBuffer (eg. through relative-read operations), so
* implementations must return a new ByteBuffer or slice for each call.
*/
@CalledByNative("I420Buffer") ByteBuffer getDataU();
/**
* Returns a direct ByteBuffer containing V-plane data. The buffer capacity is at least
* getStrideV() * ((getHeight() + 1) / 2) bytes. The position of the returned buffer is ignored
* and must be 0. Callers may mutate the ByteBuffer (eg. through relative-read operations), so
* implementations must return a new ByteBuffer or slice for each call.
*/
@CalledByNative("I420Buffer") ByteBuffer getDataV();
@CalledByNative("I420Buffer") int getStrideY();
@CalledByNative("I420Buffer") int getStrideU();
@CalledByNative("I420Buffer") int getStrideV();
}
/**
* Interface for buffers that are stored as a single texture, either in OES or RGB format.
*/
public interface TextureBuffer extends Buffer {
enum Type {
OES(GLES11Ext.GL_TEXTURE_EXTERNAL_OES),
RGB(GLES20.GL_TEXTURE_2D);
private final int glTarget;
private Type(final int glTarget) {
this.glTarget = glTarget;
}
public int getGlTarget() {
return glTarget;
}
}
Type getType();
int getTextureId();
/**
* Retrieve the transform matrix associated with the frame. This transform matrix maps 2D
* homogeneous coordinates of the form (s, t, 1) with s and t in the inclusive range [0, 1] to
* the coordinate that should be used to sample that location from the buffer.
*/
Matrix getTransformMatrix();
/**
* Create a new TextureBufferImpl with an applied transform matrix and a new size. The existing
* buffer is unchanged. The given transform matrix is applied first when texture coordinates are
* still in the unmodified [0, 1] range.
*/
default TextureBuffer applyTransformMatrix(
Matrix transformMatrix, int newWidth, int newHeight) {
throw new UnsupportedOperationException("Not implemented");
}
/**
* Returns the width of the texture in memory. This should only be used for downscaling, and you
* should still respect the width from getWidth().
*/
default public int getUnscaledWidth() {
return getWidth();
}
/**
* Returns the height of the texture in memory. This should only be used for downscaling, and
* you should still respect the height from getHeight().
*/
default public int getUnscaledHeight() {
return getHeight();
}
}
private final Buffer buffer;
private final int rotation;
private final long timestampNs;
/**
* Constructs a new VideoFrame backed by the given {@code buffer}.
*
* @note Ownership of the buffer object is tranferred to the new VideoFrame.
*/
@CalledByNative
public VideoFrame(Buffer buffer, int rotation, long timestampNs) {
if (buffer == null) {
throw new IllegalArgumentException("buffer not allowed to be null");
}
if (rotation % 90 != 0) {
throw new IllegalArgumentException("rotation must be a multiple of 90");
}
this.buffer = buffer;
this.rotation = rotation;
this.timestampNs = timestampNs;
}
@CalledByNative
public Buffer getBuffer() {
return buffer;
}
/**
* Rotation of the frame in degrees.
*/
@CalledByNative
public int getRotation() {
return rotation;
}
/**
* Timestamp of the frame in nano seconds.
*/
@CalledByNative
public long getTimestampNs() {
return timestampNs;
}
public int getRotatedWidth() {
if (rotation % 180 == 0) {
return buffer.getWidth();
}
return buffer.getHeight();
}
public int getRotatedHeight() {
if (rotation % 180 == 0) {
return buffer.getHeight();
}
return buffer.getWidth();
}
@Override
public void retain() {
buffer.retain();
}
@Override
@CalledByNative
public void release() {
buffer.release();
}
}
|