public void update() {
if (textureUpdated) {
textureUpdated = false;
surfaceTexture.updateTexImage();
// cameraTexture.getTransformMatrix(mtx);
}
}
/**
* Prepares OpenGL ES before we draw a frame.
*
* @param headTransform The head transformation in the new frame.
*/
@Override
public void onNewFrame(HeadTransform headTransform) {
// GLES20.glUseProgram(mGlProgram);
//
// mModelViewProjectionParam = GLES20.glGetUniformLocation(mGlProgram, "u_MVP");
// mLightPosParam = GLES20.glGetUniformLocation(mGlProgram, "u_LightPos");
// mModelViewParam = GLES20.glGetUniformLocation(mGlProgram, "u_MVMatrix");
// mModelParam = GLES20.glGetUniformLocation(mGlProgram, "u_Model");
// mIsFloorParam = GLES20.glGetUniformLocation(mGlProgram, "u_IsFloor");
//
// // Build the Model part of the ModelView matrix.
// Matrix.rotateM(mModelCube, 0, TIME_DELTA, 0.5f, 0.5f, 1.0f);
//
// // Build the camera matrix and apply it to the ModelView.
// Matrix.setLookAtM(mCamera, 0, 0.0f, 0.0f, CAMERA_Z, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f,
// 0.0f);
//
// headTransform.getHeadView(mHeadView, 0);
//
// checkGLError("onReadyToDraw");
float[] mtx = new float[16];
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT);
surface.updateTexImage();
surface.getTransformMatrix(mtx);
}
/**
* Latches the next buffer into the texture. Must be called from the thread that created the
* OutputSurface object.
*/
public void awaitNewImage() {
final int TIMEOUT_MS = 2500;
synchronized (frameSyncObject) {
while (!frameAvailable) {
try {
// Wait for onFrameAvailable() to signal us. Use a timeout
// to avoid stalling the test if it doesn't arrive.
frameSyncObject.wait(TIMEOUT_MS);
if (!frameAvailable) {
// TODO: if "spurious wakeup", continue while loop
throw new RuntimeException("Camera frame wait timed out");
}
} catch (InterruptedException ie) {
// shouldn't happen
throw new RuntimeException(ie);
}
}
frameAvailable = false;
}
// Latch the data.
textureRender.checkGlError("before updateTexImage");
surfaceTexture.updateTexImage();
}
/**
* Prepares OpenGL ES before we draw a frame.
*
* @param headTransform The head transformation in the new frame.
*/
@Override
public void onNewFrame(HeadTransform headTransform) {
float[] mtx = new float[16];
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT);
surface.updateTexImage();
surface.getTransformMatrix(mtx);
}
@Override
public void onDrawFrame(final GL10 gl) {
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT);
runAll(mRunOnDraw);
mFilter.onDraw(mGLTextureId, mGLCubeBuffer, mGLTextureBuffer);
runAll(mRunOnDrawEnd);
if (mSurfaceTexture != null) {
mSurfaceTexture.updateTexImage();
}
}
public void onDrawFrame(GL10 glUnused) {
synchronized (this) {
if (updateSurface) {
mSurface.updateTexImage();
mSurface.getTransformMatrix(mSTMatrix);
updateSurface = false;
} else {
return;
}
}
GLES20.glClearColor(255.0f, 255.0f, 255.0f, 1.0f);
GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT);
GLES20.glUseProgram(mProgram);
checkGlError("glUseProgram");
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);
mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
GLES20.glVertexAttribPointer(
maPositionHandle,
3,
GLES20.GL_FLOAT,
false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES,
mTriangleVertices);
checkGlError("glVertexAttribPointer maPosition");
GLES20.glEnableVertexAttribArray(maPositionHandle);
checkGlError("glEnableVertexAttribArray maPositionHandle");
mTextureVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET);
GLES20.glVertexAttribPointer(
maTextureHandle,
2,
GLES20.GL_FLOAT,
false,
TEXTURE_VERTICES_DATA_STRIDE_BYTES,
mTextureVertices);
checkGlError("glVertexAttribPointer maTextureHandle");
GLES20.glEnableVertexAttribArray(maTextureHandle);
checkGlError("glEnableVertexAttribArray maTextureHandle");
Matrix.setIdentityM(mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
checkGlError("glDrawArrays");
GLES20.glFinish();
}
private boolean captureScreenshotTextureAndSetViewport() {
if (!attachEglContext()) {
return false;
}
try {
if (!mTexNamesGenerated) {
GLES10.glGenTextures(1, mTexNames, 0);
if (checkGlErrors("glGenTextures")) {
return false;
}
mTexNamesGenerated = true;
}
final SurfaceTexture st = new SurfaceTexture(mTexNames[0]);
final Surface s = new Surface(st);
try {
SurfaceControl.screenshot(
SurfaceControl.getBuiltInDisplay(SurfaceControl.BUILT_IN_DISPLAY_ID_MAIN), s);
} finally {
s.release();
}
st.updateTexImage();
st.getTransformMatrix(mTexMatrix);
// Set up texture coordinates for a quad.
// We might need to change this if the texture ends up being
// a different size from the display for some reason.
mTexCoordBuffer.put(0, 0f);
mTexCoordBuffer.put(1, 0f);
mTexCoordBuffer.put(2, 0f);
mTexCoordBuffer.put(3, 1f);
mTexCoordBuffer.put(4, 1f);
mTexCoordBuffer.put(5, 1f);
mTexCoordBuffer.put(6, 1f);
mTexCoordBuffer.put(7, 0f);
// Set up our viewport.
GLES10.glViewport(0, 0, mDisplayWidth, mDisplayHeight);
GLES10.glMatrixMode(GLES10.GL_PROJECTION);
GLES10.glLoadIdentity();
GLES10.glOrthof(0, mDisplayWidth, 0, mDisplayHeight, 0, 1);
GLES10.glMatrixMode(GLES10.GL_MODELVIEW);
GLES10.glLoadIdentity();
GLES10.glMatrixMode(GLES10.GL_TEXTURE);
GLES10.glLoadIdentity();
GLES10.glLoadMatrixf(mTexMatrix, 0);
} finally {
detachEglContext();
}
return true;
}
// Need to call from GL Thread.
// To update texture image if there is a new available frame from video source.
public boolean applyUpdate() {
if (mSurfaceTexture == null) {
return false;
}
if (!mIsNewFrameArrival.get()) {
return false;
}
mIsNewFrameArrival.set(false);
mSurfaceTexture.updateTexImage();
onSurfaceTextureUpdated(mSurfaceTexture);
if (isFirstFrameAndPause()) {
pause();
return true;
}
return false;
}
@Override
public void process(FilterContext context) {
if (mLogVerbose) Log.v(TAG, "Processing new frame");
// First, get new frame if available
if (mWaitForNewFrame || mFirstFrame) {
boolean gotNewFrame;
if (mWaitTimeout != 0) {
gotNewFrame = mNewFrameAvailable.block(mWaitTimeout);
if (!gotNewFrame) {
if (!mCloseOnTimeout) {
throw new RuntimeException("Timeout waiting for new frame");
} else {
if (mLogVerbose) Log.v(TAG, "Timeout waiting for a new frame. Closing.");
closeOutputPort("video");
return;
}
}
} else {
mNewFrameAvailable.block();
}
mNewFrameAvailable.close();
mFirstFrame = false;
}
mSurfaceTexture.updateTexImage();
mSurfaceTexture.getTransformMatrix(mFrameTransform);
Matrix.multiplyMM(
mMappedCoords, 0,
mFrameTransform, 0,
mSourceCoords, 0);
mFrameExtractor.setSourceRegion(
mMappedCoords[0], mMappedCoords[1],
mMappedCoords[4], mMappedCoords[5],
mMappedCoords[8], mMappedCoords[9],
mMappedCoords[12], mMappedCoords[13]);
// Next, render to output
Frame output = context.getFrameManager().newFrame(mOutputFormat);
mFrameExtractor.process(mMediaFrame, output);
output.setTimestamp(mSurfaceTexture.getTimestamp());
pushOutput("video", output);
output.release();
}
@Override
public void draw(GLCanvas canvas, int x, int y, int width, int height) {
synchronized (this) {
if (!mHasTexture) return;
mSurfaceTexture.updateTexImage();
mSurfaceTexture.getTransformMatrix(mTransform);
// Flip vertically.
canvas.save(GLCanvas.SAVE_FLAG_MATRIX);
int cx = x + width / 2;
int cy = y + height / 2;
canvas.translate(cx, cy);
canvas.scale(1, -1, 1);
canvas.translate(-cx, -cy);
canvas.drawTexture(mExtTexture, mTransform, x, y, width, height);
canvas.restore();
}
}
@Override
public void onNewFrame(HeadTransform headTransform) {
Matrix.rotateM(mModelCube, 0, TIME_DELTA, 0.5f, 0.5f, 1.0f);
// Build the camera matrix and apply it to the ModelView.
Matrix.setLookAtM(mCamera, 0, 0.0f, 0.0f, CAMERA_Z, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f);
headTransform.getHeadView(headView, 0);
checkGLError("onReadyToDraw");
float[] mtx = new float[16];
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT);
surface.updateTexImage();
surface.getTransformMatrix(mtx);
mHeadTransform = headTransform;
headTransform.getHeadView(headView, 0);
}
/** Tells the VideoPlayerHelper to update the data from the video feed */
@SuppressLint("NewApi")
public byte updateVideoData() {
if (!isPlayableOnTexture()) {
// DebugLog.LOGD("Cannot update the data of this video since it is not on texture");
return -1;
}
byte result = -1;
mSurfaceTextureLock.lock();
if (mSurfaceTexture != null) {
// Only request an update if currently playing
if (mCurrentState == MEDIA_STATE.PLAYING) mSurfaceTexture.updateTexImage();
result = mTextureID;
}
mSurfaceTextureLock.unlock();
return result;
}
@Override
public void onDrawFrame(GL10 gl) {
// draws the skybox
if (mUseSkybox) super.onDrawFrame(gl);
// refresh camera texture
if (mCameraFrameAvaible) {
mCameraSurfaceTex.updateTexImage();
mCameraFrameAvaible = false;
}
mCameraSurfaceTex.setOnFrameAvailableListener(this);
// draw camera surface
gl.glEnable(GLES11Ext.GL_TEXTURE_EXTERNAL_OES);
mCameraSurface.draw(gl, mViewMatrix);
gl.glDisable(GLES11Ext.GL_TEXTURE_EXTERNAL_OES);
// draw the viewFinder
mViewFinder.draw(gl, mViewMatrix);
// launch memory cleanup??
Runtime info = Runtime.getRuntime();
long freeMem = info.freeMemory() / 1048576L;
if (freeMem < MEMORY_CLEANUP_THRESHOLD) mHasToFreeMemory = true;
else mHasToFreeMemory = false;
// the snapshots that are in FOV
mSnapshotsLock.lock();
for (Snapshot3D snap : mSnapshots) {
float distance = this.getSnapshotDisnance(snap);
if (mHasToFreeMemory && distance > AUTO_UNLOADTEXTURE_ANGLE) {
snap.unloadGLTexture(gl);
} else if (this.getSnapshotDisnance(snap) < AUTO_LOADTEXTURE_ANGLE) {
snap.loadGLTexture(gl);
}
if (distance > 120.0f) snap.setVisible(false);
else {
snap.setVisible(true);
snap.draw(gl, super.getRotationMatrix());
}
}
mSnapshotsLock.unlock();
// ... and then all markers with newly computed alpha
float d;
// draw markers
if (mUseMarkers) {
mTargetsLock.lock();
for (Snapshot3D dot : mDots) {
d = getSnapshotDisnance(dot);
if (d > 60.0f) {
dot.setVisible(false);
} else {
dot.setVisible(true);
// Set alpha based on camera distance to the point
d = d * mMarkersAttenuationFactor / 360.0f;
d = (d > 1.0f ? 1.0f : d);
dot.setAlpha(1.0f - d);
dot.draw(gl, super.getRotationMatrix());
}
}
mTargetsLock.unlock();
}
if (mUseContours) {
mTargetsLock.lock();
for (Snapshot3D contour : mContours) {
d = getSnapshotDisnance(contour);
if (d > 60.0f) {
contour.setVisible(false);
} else {
contour.setVisible(true);
// Set alpha based on camera distance to the point
d = d * mMarkersAttenuationFactor / 360.0f;
d = (d > 1.0f ? 1.0f : d);
contour.setAlpha(1.0f - d);
contour.draw(gl, super.getRotationMatrix());
}
}
mTargetsLock.unlock();
}
}
@Override
public void onDrawFrame(GL10 unused) {
if (VERBOSE) Log.d(TAG, "onDrawFrame tex=" + mTextureId);
boolean showBox = false;
// Latch the latest frame. If there isn't anything new, we'll just re-use whatever
// was there before.
mSurfaceTexture.updateTexImage();
// If the recording state is changing, take care of it here. Ideally we wouldn't
// be doing all this in onDrawFrame(), but the EGLContext sharing with GLSurfaceView
// makes it hard to do elsewhere.
if (mRecordingEnabled) {
switch (mRecordingStatus) {
case RECORDING_OFF:
Log.d(TAG, "START recording");
// start recording
mVideoEncoder.startRecording(
new TextureMovieEncoder.EncoderConfig(
mOutputFile, 480, 480, 1000000, EGL14.eglGetCurrentContext()));
// TODO: get optimal width and height according to specified devices
mRecordingStatus = RECORDING_ON;
break;
case RECORDING_RESUMED:
Log.d(TAG, "RESUME recording");
mVideoEncoder.updateSharedContext(EGL14.eglGetCurrentContext());
mRecordingStatus = RECORDING_ON;
break;
case RECORDING_ON:
// yay
break;
default:
throw new RuntimeException("unknown status " + mRecordingStatus);
}
} else {
switch (mRecordingStatus) {
case RECORDING_ON:
case RECORDING_RESUMED:
// stop recording
Log.d(TAG, "STOP recording");
mVideoEncoder.stopRecording();
mRecordingStatus = RECORDING_OFF;
break;
case RECORDING_OFF:
// yay
break;
default:
throw new RuntimeException("unknown status " + mRecordingStatus);
}
}
// Set the video encoder's texture name. We only need to do this once, but in the
// current implementation it has to happen after the video encoder is started, so
// we just do it here.
//
// TODO: be less lame.
mVideoEncoder.setTextureId(mTextureId);
// Tell the video encoder thread that a new frame is available.
// This will be ignored if we're not actually recording.
mVideoEncoder.frameAvailable(mSurfaceTexture);
if (mIncomingWidth <= 0 || mIncomingHeight <= 0) {
// Texture size isn't set yet. This is only used for the filters, but to be
// safe we can just skip drawing while we wait for the various races to resolve.
// (This seems to happen if you toggle the screen off/on with power button.)
Log.i(TAG, "Drawing before incoming texture size set; skipping");
return;
}
// Update the filter, if necessary.
if (mCurrentFilter != mNewFilter) {
updateFilter();
}
if (mIncomingSizeUpdated) {
mFullScreen.getProgram().setTexSize(mIncomingWidth, mIncomingHeight);
mIncomingSizeUpdated = false;
}
// Draw the video frame.
mSurfaceTexture.getTransformMatrix(mSTMatrix);
mFullScreen.drawFrame(mTextureId, mSTMatrix);
// Draw a flashing box if we're recording. This only appears on screen.
showBox = (mRecordingStatus == RECORDING_ON);
if (showBox && (++mFrameCount & 0x04) == 0) {
drawBox();
}
}
private void draw(GlRectDrawer drawer) {
if (!seenFrame) {
// No frame received yet - nothing to render.
return;
}
long now = System.nanoTime();
final boolean isNewFrame;
synchronized (pendingFrameLock) {
isNewFrame = (pendingFrame != null);
if (isNewFrame && startTimeNs == -1) {
startTimeNs = now;
}
if (isNewFrame) {
if (pendingFrame.yuvFrame) {
rendererType = RendererType.RENDERER_YUV;
drawer.uploadYuvData(
yuvTextures,
pendingFrame.width,
pendingFrame.height,
pendingFrame.yuvStrides,
pendingFrame.yuvPlanes);
// The convention in WebRTC is that the first element in a ByteBuffer corresponds to the
// top-left corner of the image, but in glTexImage2D() the first element corresponds to
// the bottom-left corner. We correct this discrepancy by setting a vertical flip as
// sampling matrix.
final float[] samplingMatrix = RendererCommon.verticalFlipMatrix();
rotatedSamplingMatrix =
RendererCommon.rotateTextureMatrix(samplingMatrix, pendingFrame.rotationDegree);
} else {
rendererType = RendererType.RENDERER_TEXTURE;
// External texture rendering. Update texture image to latest and make a deep copy of
// the external texture.
// TODO(magjed): Move updateTexImage() to the video source instead.
final SurfaceTexture surfaceTexture = (SurfaceTexture) pendingFrame.textureObject;
surfaceTexture.updateTexImage();
final float[] samplingMatrix = new float[16];
surfaceTexture.getTransformMatrix(samplingMatrix);
rotatedSamplingMatrix =
RendererCommon.rotateTextureMatrix(samplingMatrix, pendingFrame.rotationDegree);
// Reallocate offscreen texture if necessary.
textureCopy.setSize(pendingFrame.rotatedWidth(), pendingFrame.rotatedHeight());
// Bind our offscreen framebuffer.
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, textureCopy.getFrameBufferId());
GlUtil.checkNoGLES2Error("glBindFramebuffer");
// Copy the OES texture content. This will also normalize the sampling matrix.
GLES20.glViewport(0, 0, textureCopy.getWidth(), textureCopy.getHeight());
drawer.drawOes(pendingFrame.textureId, rotatedSamplingMatrix);
rotatedSamplingMatrix = RendererCommon.identityMatrix();
// Restore normal framebuffer.
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
}
copyTimeNs += (System.nanoTime() - now);
VideoRenderer.renderFrameDone(pendingFrame);
pendingFrame = null;
}
}
// OpenGL defaults to lower left origin - flip vertically.
GLES20.glViewport(
displayLayout.left,
screenHeight - displayLayout.bottom,
displayLayout.width(),
displayLayout.height());
updateLayoutMatrix();
final float[] texMatrix =
RendererCommon.multiplyMatrices(rotatedSamplingMatrix, layoutMatrix);
if (rendererType == RendererType.RENDERER_YUV) {
drawer.drawYuv(yuvTextures, texMatrix);
} else {
drawer.drawRgb(textureCopy.getTextureId(), texMatrix);
}
if (isNewFrame) {
framesRendered++;
drawTimeNs += (System.nanoTime() - now);
if ((framesRendered % 300) == 0) {
logStatistics();
}
}
}
