@SuppressLint("NewApi")
@Override
public void onSurfaceCreated(GL10 unused, EGLConfig config) {
Logging.d(TAG, "VideoRendererGui.onSurfaceCreated");
// Store render EGL context.
if (CURRENT_SDK_VERSION >= EGL14_SDK_VERSION) {
synchronized (VideoRendererGui.class) {
eglContext = EGL14.eglGetCurrentContext();
Logging.d(TAG, "VideoRendererGui EGL Context: " + eglContext);
}
}
synchronized (yuvImageRenderers) {
// Create drawer for YUV/OES frames.
drawer = new GlRectDrawer();
// Create textures for all images.
for (YuvImageRenderer yuvImageRenderer : yuvImageRenderers) {
yuvImageRenderer.createTextures();
}
onSurfaceCreatedCalled = true;
}
GlUtil.checkNoGLES2Error("onSurfaceCreated done");
GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1);
GLES20.glClearColor(0.15f, 0.15f, 0.15f, 1.0f);
// Fire EGL context ready event.
synchronized (VideoRendererGui.class) {
if (eglContextReady != null) {
eglContextReady.run();
}
}
}
/**
* Creates a test texture in the current GL context.
*
* <p>This follows image conventions, so the pixel data at offset zero is intended to appear in
* the top-left corner. Color values for non-opaque alpha will be pre-multiplied.
*
* @return Handle to texture.
*/
public static int createTestTexture(Image which) {
ByteBuffer buf;
switch (which) {
case COARSE:
buf = sCoarseImageData;
break;
case FINE:
buf = sFineImageData;
break;
default:
throw new RuntimeException("unknown image");
}
return GlUtil.createImageTexture(buf, TEX_SIZE, TEX_SIZE, FORMAT);
}
private void createTextures() {
Logging.d(
TAG,
" YuvImageRenderer.createTextures "
+ id
+ " on GL thread:"
+ Thread.currentThread().getId());
// Generate 3 texture ids for Y/U/V and place them into |yuvTextures|.
for (int i = 0; i < 3; i++) {
yuvTextures[i] = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D);
}
// Generate texture and framebuffer for offscreen texture copy.
textureCopy = new GlTextureFrameBuffer(GLES20.GL_RGB);
}
/**
* Saves the EGL surface to a file.
*
* <p>Expects that this object's EGL surface is current.
*/
public void saveFrame(File file) throws IOException {
if (!mEglCore.isCurrent(mEGLSurface)) {
throw new RuntimeException("Expected EGL context/surface is not current");
}
// glReadPixels fills in a "direct" ByteBuffer with what is essentially big-endian RGBA
// data (i.e. a byte of red, followed by a byte of green...). While the Bitmap
// constructor that takes an int[] wants little-endian ARGB (blue/red swapped), the
// Bitmap "copy pixels" method wants the same format GL provides.
//
// Ideally we'd have some way to re-use the ByteBuffer, especially if we're calling
// here often.
//
// Making this even more interesting is the upside-down nature of GL, which means
// our output will look upside down relative to what appears on screen if the
// typical GL conventions are used.
String filename = file.toString();
int width = getWidth();
int height = getHeight();
ByteBuffer buf = ByteBuffer.allocateDirect(width * height * 4);
buf.order(ByteOrder.LITTLE_ENDIAN);
GLES20.glReadPixels(0, 0, width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, buf);
GlUtil.checkGlError("glReadPixels");
buf.rewind();
BufferedOutputStream bos = null;
try {
bos = new BufferedOutputStream(new FileOutputStream(filename));
Bitmap bmp = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888);
bmp.copyPixelsFromBuffer(buf);
bmp.compress(Bitmap.CompressFormat.PNG, 90, bos);
bmp.recycle();
} finally {
if (bos != null) bos.close();
}
Log.d(TAG, "Saved " + width + "x" + height + " frame as '" + filename + "'");
}
// Pass null in |sharedContext| to configure the codec for ByteBuffer output.
private boolean initDecode(VideoCodecType type, int width, int height, EGLContext sharedContext) {
if (mediaCodecThread != null) {
throw new RuntimeException("Forgot to release()?");
}
useSurface = (sharedContext != null);
String mime = null;
String[] supportedCodecPrefixes = null;
if (type == VideoCodecType.VIDEO_CODEC_VP8) {
mime = VP8_MIME_TYPE;
supportedCodecPrefixes = supportedVp8HwCodecPrefixes;
} else if (type == VideoCodecType.VIDEO_CODEC_H264) {
mime = H264_MIME_TYPE;
supportedCodecPrefixes = supportedH264HwCodecPrefixes;
} else {
throw new RuntimeException("Non supported codec " + type);
}
DecoderProperties properties = findDecoder(mime, supportedCodecPrefixes);
if (properties == null) {
throw new RuntimeException("Cannot find HW decoder for " + type);
}
Logging.d(
TAG,
"Java initDecode: "
+ type
+ " : "
+ width
+ " x "
+ height
+ ". Color: 0x"
+ Integer.toHexString(properties.colorFormat)
+ ". Use Surface: "
+ useSurface);
if (sharedContext != null) {
Logging.d(TAG, "Decoder shared EGL Context: " + sharedContext);
}
mediaCodecThread = Thread.currentThread();
try {
Surface decodeSurface = null;
this.width = width;
this.height = height;
stride = width;
sliceHeight = height;
if (useSurface) {
// Create shared EGL context.
eglBase = new EglBase(sharedContext, EglBase.ConfigType.PIXEL_BUFFER);
eglBase.createDummyPbufferSurface();
eglBase.makeCurrent();
// Create output surface
textureID = GlUtil.generateTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES);
Logging.d(TAG, "Video decoder TextureID = " + textureID);
surfaceTexture = new SurfaceTexture(textureID);
surface = new Surface(surfaceTexture);
decodeSurface = surface;
}
MediaFormat format = MediaFormat.createVideoFormat(mime, width, height);
if (!useSurface) {
format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat);
}
Logging.d(TAG, " Format: " + format);
mediaCodec = MediaCodecVideoEncoder.createByCodecName(properties.codecName);
if (mediaCodec == null) {
return false;
}
mediaCodec.configure(format, decodeSurface, null, 0);
mediaCodec.start();
colorFormat = properties.colorFormat;
outputBuffers = mediaCodec.getOutputBuffers();
inputBuffers = mediaCodec.getInputBuffers();
Logging.d(
TAG,
"Input buffers: " + inputBuffers.length + ". Output buffers: " + outputBuffers.length);
return true;
} catch (IllegalStateException e) {
Logging.e(TAG, "initDecode failed", e);
return false;
}
}
/**
* Test disconnecting the SurfaceTextureHelper while holding a pending texture frame. The pending
* texture frame should still be valid, and this is tested by drawing the texture frame to a pixel
* buffer and reading it back with glReadPixels().
*/
@MediumTest
public static void testLateReturnFrame() throws InterruptedException {
final int width = 16;
final int height = 16;
// Create EGL base with a pixel buffer as display output.
final EglBase eglBase = EglBase.create(null, EglBase.ConfigType.PIXEL_BUFFER);
eglBase.createPbufferSurface(width, height);
// Create SurfaceTextureHelper and listener.
final SurfaceTextureHelper surfaceTextureHelper =
SurfaceTextureHelper.create(eglBase.getEglBaseContext());
final MockTextureListener listener = new MockTextureListener();
surfaceTextureHelper.setListener(listener);
surfaceTextureHelper.getSurfaceTexture().setDefaultBufferSize(width, height);
// Create resources for stubbing an OES texture producer. |eglOesBase| has the SurfaceTexture in
// |surfaceTextureHelper| as the target EGLSurface.
final EglBase eglOesBase =
EglBase.create(eglBase.getEglBaseContext(), EglBase.ConfigType.PLAIN);
eglOesBase.createSurface(surfaceTextureHelper.getSurfaceTexture());
assertEquals(eglOesBase.surfaceWidth(), width);
assertEquals(eglOesBase.surfaceHeight(), height);
final int red = 79;
final int green = 66;
final int blue = 161;
// Draw a constant color frame onto the SurfaceTexture.
eglOesBase.makeCurrent();
GLES20.glClearColor(red / 255.0f, green / 255.0f, blue / 255.0f, 1.0f);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
// swapBuffers() will ultimately trigger onTextureFrameAvailable().
eglOesBase.swapBuffers();
eglOesBase.release();
// Wait for OES texture frame.
listener.waitForNewFrame();
// Diconnect while holding the frame.
surfaceTextureHelper.disconnect();
// Draw the pending texture frame onto the pixel buffer.
eglBase.makeCurrent();
final GlRectDrawer drawer = new GlRectDrawer();
drawer.drawOes(listener.oesTextureId, listener.transformMatrix);
drawer.release();
// Download the pixels in the pixel buffer as RGBA. Not all platforms support RGB, e.g. Nexus 9.
final ByteBuffer rgbaData = ByteBuffer.allocateDirect(width * height * 4);
GLES20.glReadPixels(0, 0, width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, rgbaData);
GlUtil.checkNoGLES2Error("glReadPixels");
eglBase.release();
// Assert rendered image is expected constant color.
while (rgbaData.hasRemaining()) {
assertEquals(rgbaData.get() & 0xFF, red);
assertEquals(rgbaData.get() & 0xFF, green);
assertEquals(rgbaData.get() & 0xFF, blue);
assertEquals(rgbaData.get() & 0xFF, 255);
}
// Late frame return after everything has been disconnected and released.
surfaceTextureHelper.returnTextureFrame();
}
private void draw(GlRectDrawer drawer) {
if (!seenFrame) {
// No frame received yet - nothing to render.
return;
}
long now = System.nanoTime();
final boolean isNewFrame;
synchronized (pendingFrameLock) {
isNewFrame = (pendingFrame != null);
if (isNewFrame && startTimeNs == -1) {
startTimeNs = now;
}
if (isNewFrame) {
if (pendingFrame.yuvFrame) {
rendererType = RendererType.RENDERER_YUV;
drawer.uploadYuvData(
yuvTextures,
pendingFrame.width,
pendingFrame.height,
pendingFrame.yuvStrides,
pendingFrame.yuvPlanes);
// The convention in WebRTC is that the first element in a ByteBuffer corresponds to the
// top-left corner of the image, but in glTexImage2D() the first element corresponds to
// the bottom-left corner. We correct this discrepancy by setting a vertical flip as
// sampling matrix.
final float[] samplingMatrix = RendererCommon.verticalFlipMatrix();
rotatedSamplingMatrix =
RendererCommon.rotateTextureMatrix(samplingMatrix, pendingFrame.rotationDegree);
} else {
rendererType = RendererType.RENDERER_TEXTURE;
// External texture rendering. Update texture image to latest and make a deep copy of
// the external texture.
// TODO(magjed): Move updateTexImage() to the video source instead.
final SurfaceTexture surfaceTexture = (SurfaceTexture) pendingFrame.textureObject;
surfaceTexture.updateTexImage();
final float[] samplingMatrix = new float[16];
surfaceTexture.getTransformMatrix(samplingMatrix);
rotatedSamplingMatrix =
RendererCommon.rotateTextureMatrix(samplingMatrix, pendingFrame.rotationDegree);
// Reallocate offscreen texture if necessary.
textureCopy.setSize(pendingFrame.rotatedWidth(), pendingFrame.rotatedHeight());
// Bind our offscreen framebuffer.
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, textureCopy.getFrameBufferId());
GlUtil.checkNoGLES2Error("glBindFramebuffer");
// Copy the OES texture content. This will also normalize the sampling matrix.
GLES20.glViewport(0, 0, textureCopy.getWidth(), textureCopy.getHeight());
drawer.drawOes(pendingFrame.textureId, rotatedSamplingMatrix);
rotatedSamplingMatrix = RendererCommon.identityMatrix();
// Restore normal framebuffer.
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
}
copyTimeNs += (System.nanoTime() - now);
VideoRenderer.renderFrameDone(pendingFrame);
pendingFrame = null;
}
}
// OpenGL defaults to lower left origin - flip vertically.
GLES20.glViewport(
displayLayout.left,
screenHeight - displayLayout.bottom,
displayLayout.width(),
displayLayout.height());
updateLayoutMatrix();
final float[] texMatrix =
RendererCommon.multiplyMatrices(rotatedSamplingMatrix, layoutMatrix);
if (rendererType == RendererType.RENDERER_YUV) {
drawer.drawYuv(yuvTextures, texMatrix);
} else {
drawer.drawRgb(textureCopy.getTextureId(), texMatrix);
}
if (isNewFrame) {
framesRendered++;
drawTimeNs += (System.nanoTime() - now);
if ((framesRendered % 300) == 0) {
logStatistics();
}
}
}
