1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259
|
/*
* Copyright 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
package org.webrtc;
import android.graphics.Point;
import android.opengl.Matrix;
import android.view.View;
/**
* Static helper functions for renderer implementations.
*/
public class RendererCommon {
/** Interface for reporting rendering events. */
public static interface RendererEvents {
/**
* Callback fired once first frame is rendered.
*/
public void onFirstFrameRendered();
/**
* Callback fired when rendered frame resolution or rotation has changed.
*/
public void onFrameResolutionChanged(int videoWidth, int videoHeight, int rotation);
}
/**
* Interface for rendering frames on an EGLSurface with specified viewport location. Rotation,
* mirror, and cropping is specified using a 4x4 texture coordinate transform matrix. The frame
* input can either be an OES texture, RGB texture, or YUV textures in I420 format. The function
* release() must be called manually to free the resources held by this object.
*/
public static interface GlDrawer {
/**
* Functions for drawing frames with different sources. The rendering surface target is
* implied by the current EGL context of the calling thread and requires no explicit argument.
* The coordinates specify the viewport location on the surface target.
*/
void drawOes(int oesTextureId, float[] texMatrix, int frameWidth, int frameHeight,
int viewportX, int viewportY, int viewportWidth, int viewportHeight);
void drawRgb(int textureId, float[] texMatrix, int frameWidth, int frameHeight, int viewportX,
int viewportY, int viewportWidth, int viewportHeight);
void drawYuv(int[] yuvTextures, float[] texMatrix, int frameWidth, int frameHeight,
int viewportX, int viewportY, int viewportWidth, int viewportHeight);
/**
* Release all GL resources. This needs to be done manually, otherwise resources may leak.
*/
void release();
}
/**
* Helper class for determining layout size based on layout requirements, scaling type, and video
* aspect ratio.
*/
public static class VideoLayoutMeasure {
// The scaling type determines how the video will fill the allowed layout area in measure(). It
// can be specified separately for the case when video has matched orientation with layout size
// and when there is an orientation mismatch.
private float visibleFractionMatchOrientation =
convertScalingTypeToVisibleFraction(ScalingType.SCALE_ASPECT_BALANCED);
private float visibleFractionMismatchOrientation =
convertScalingTypeToVisibleFraction(ScalingType.SCALE_ASPECT_BALANCED);
public void setScalingType(ScalingType scalingType) {
setScalingType(/* scalingTypeMatchOrientation= */ scalingType,
/* scalingTypeMismatchOrientation= */ scalingType);
}
public void setScalingType(
ScalingType scalingTypeMatchOrientation, ScalingType scalingTypeMismatchOrientation) {
this.visibleFractionMatchOrientation =
convertScalingTypeToVisibleFraction(scalingTypeMatchOrientation);
this.visibleFractionMismatchOrientation =
convertScalingTypeToVisibleFraction(scalingTypeMismatchOrientation);
}
public void setVisibleFraction(
float visibleFractionMatchOrientation, float visibleFractionMismatchOrientation) {
this.visibleFractionMatchOrientation = visibleFractionMatchOrientation;
this.visibleFractionMismatchOrientation = visibleFractionMismatchOrientation;
}
public Point measure(int widthSpec, int heightSpec, int frameWidth, int frameHeight) {
// Calculate max allowed layout size.
final int maxWidth = View.getDefaultSize(Integer.MAX_VALUE, widthSpec);
final int maxHeight = View.getDefaultSize(Integer.MAX_VALUE, heightSpec);
if (frameWidth == 0 || frameHeight == 0 || maxWidth == 0 || maxHeight == 0) {
return new Point(maxWidth, maxHeight);
}
// Calculate desired display size based on scaling type, video aspect ratio,
// and maximum layout size.
final float frameAspect = frameWidth / (float) frameHeight;
final float displayAspect = maxWidth / (float) maxHeight;
final float visibleFraction = (frameAspect > 1.0f) == (displayAspect > 1.0f)
? visibleFractionMatchOrientation
: visibleFractionMismatchOrientation;
final Point layoutSize = getDisplaySize(visibleFraction, frameAspect, maxWidth, maxHeight);
// If the measure specification is forcing a specific size - yield.
if (View.MeasureSpec.getMode(widthSpec) == View.MeasureSpec.EXACTLY) {
layoutSize.x = maxWidth;
}
if (View.MeasureSpec.getMode(heightSpec) == View.MeasureSpec.EXACTLY) {
layoutSize.y = maxHeight;
}
return layoutSize;
}
}
// Types of video scaling:
// SCALE_ASPECT_FIT - video frame is scaled to fit the size of the view by
// maintaining the aspect ratio (black borders may be displayed).
// SCALE_ASPECT_FILL - video frame is scaled to fill the size of the view by
// maintaining the aspect ratio. Some portion of the video frame may be
// clipped.
// SCALE_ASPECT_BALANCED - Compromise between FIT and FILL. Video frame will fill as much as
// possible of the view while maintaining aspect ratio, under the constraint that at least
// `BALANCED_VISIBLE_FRACTION` of the frame content will be shown.
public static enum ScalingType { SCALE_ASPECT_FIT, SCALE_ASPECT_FILL, SCALE_ASPECT_BALANCED }
// The minimum fraction of the frame content that will be shown for `SCALE_ASPECT_BALANCED`.
// This limits excessive cropping when adjusting display size.
private static float BALANCED_VISIBLE_FRACTION = 0.5625f;
/**
* Returns layout transformation matrix that applies an optional mirror effect and compensates
* for video vs display aspect ratio.
*/
public static float[] getLayoutMatrix(
boolean mirror, float videoAspectRatio, float displayAspectRatio) {
float scaleX = 1;
float scaleY = 1;
// Scale X or Y dimension so that video and display size have same aspect ratio.
if (displayAspectRatio > videoAspectRatio) {
scaleY = videoAspectRatio / displayAspectRatio;
} else {
scaleX = displayAspectRatio / videoAspectRatio;
}
// Apply optional horizontal flip.
if (mirror) {
scaleX *= -1;
}
final float matrix[] = new float[16];
Matrix.setIdentityM(matrix, 0);
Matrix.scaleM(matrix, 0, scaleX, scaleY, 1);
adjustOrigin(matrix);
return matrix;
}
/** Converts a float[16] matrix array to android.graphics.Matrix. */
public static android.graphics.Matrix convertMatrixToAndroidGraphicsMatrix(float[] matrix4x4) {
// clang-format off
float[] values = {
matrix4x4[0 * 4 + 0], matrix4x4[1 * 4 + 0], matrix4x4[3 * 4 + 0],
matrix4x4[0 * 4 + 1], matrix4x4[1 * 4 + 1], matrix4x4[3 * 4 + 1],
matrix4x4[0 * 4 + 3], matrix4x4[1 * 4 + 3], matrix4x4[3 * 4 + 3],
};
// clang-format on
android.graphics.Matrix matrix = new android.graphics.Matrix();
matrix.setValues(values);
return matrix;
}
/** Converts android.graphics.Matrix to a float[16] matrix array. */
public static float[] convertMatrixFromAndroidGraphicsMatrix(android.graphics.Matrix matrix) {
float[] values = new float[9];
matrix.getValues(values);
// The android.graphics.Matrix looks like this:
// [x1 y1 w1]
// [x2 y2 w2]
// [x3 y3 w3]
// We want to contruct a matrix that looks like this:
// [x1 y1 0 w1]
// [x2 y2 0 w2]
// [ 0 0 1 0]
// [x3 y3 0 w3]
// Since it is stored in column-major order, it looks like this:
// [x1 x2 0 x3
// y1 y2 0 y3
// 0 0 1 0
// w1 w2 0 w3]
// clang-format off
float[] matrix4x4 = {
values[0 * 3 + 0], values[1 * 3 + 0], 0, values[2 * 3 + 0],
values[0 * 3 + 1], values[1 * 3 + 1], 0, values[2 * 3 + 1],
0, 0, 1, 0,
values[0 * 3 + 2], values[1 * 3 + 2], 0, values[2 * 3 + 2],
};
// clang-format on
return matrix4x4;
}
/**
* Calculate display size based on scaling type, video aspect ratio, and maximum display size.
*/
public static Point getDisplaySize(
ScalingType scalingType, float videoAspectRatio, int maxDisplayWidth, int maxDisplayHeight) {
return getDisplaySize(convertScalingTypeToVisibleFraction(scalingType), videoAspectRatio,
maxDisplayWidth, maxDisplayHeight);
}
/**
* Move `matrix` transformation origin to (0.5, 0.5). This is the origin for texture coordinates
* that are in the range 0 to 1.
*/
private static void adjustOrigin(float[] matrix) {
// Note that OpenGL is using column-major order.
// Pre translate with -0.5 to move coordinates to range [-0.5, 0.5].
matrix[12] -= 0.5f * (matrix[0] + matrix[4]);
matrix[13] -= 0.5f * (matrix[1] + matrix[5]);
// Post translate with 0.5 to move coordinates to range [0, 1].
matrix[12] += 0.5f;
matrix[13] += 0.5f;
}
/**
* Each scaling type has a one-to-one correspondence to a numeric minimum fraction of the video
* that must remain visible.
*/
private static float convertScalingTypeToVisibleFraction(ScalingType scalingType) {
switch (scalingType) {
case SCALE_ASPECT_FIT:
return 1.0f;
case SCALE_ASPECT_FILL:
return 0.0f;
case SCALE_ASPECT_BALANCED:
return BALANCED_VISIBLE_FRACTION;
default:
throw new IllegalArgumentException();
}
}
/**
* Calculate display size based on minimum fraction of the video that must remain visible,
* video aspect ratio, and maximum display size.
*/
public static Point getDisplaySize(
float minVisibleFraction, float videoAspectRatio, int maxDisplayWidth, int maxDisplayHeight) {
// If there is no constraint on the amount of cropping, fill the allowed display area.
if (minVisibleFraction == 0 || videoAspectRatio == 0) {
return new Point(maxDisplayWidth, maxDisplayHeight);
}
// Each dimension is constrained on max display size and how much we are allowed to crop.
final int width = Math.min(
maxDisplayWidth, Math.round(maxDisplayHeight / minVisibleFraction * videoAspectRatio));
final int height = Math.min(
maxDisplayHeight, Math.round(maxDisplayWidth / minVisibleFraction / videoAspectRatio));
return new Point(width, height);
}
}
|