/**
* Edits a video file, saving the contents to a new file. This involves decoding and re-encoding,
* not to mention conversions between YUV and RGB, and so may be lossy.
*
* <p>If we recognize the decoded format we can do this in Java code using the ByteBuffer[]
* output, but it's not practical to support all OEM formats. By using a SurfaceTexture for output
* and a Surface for input, we can avoid issues with obscure formats and can use a fragment shader
* to do transformations.
*/
private VideoChunks editVideoFile(VideoChunks inputData) {
if (VERBOSE) Log.d(TAG, "editVideoFile " + mWidth + "x" + mHeight);
VideoChunks outputData = new VideoChunks();
MediaCodec decoder = null;
MediaCodec encoder = null;
InputSurface inputSurface = null;
OutputSurface outputSurface = null;
try {
MediaFormat inputFormat = inputData.getMediaFormat();
// Create an encoder format that matches the input format. (Might be able to just
// re-use the format used to generate the video, since we want it to be the same.)
MediaFormat outputFormat = MediaFormat.createVideoFormat(MIME_TYPE, mWidth, mHeight);
outputFormat.setInteger(
MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
outputFormat.setInteger(
MediaFormat.KEY_BIT_RATE, inputFormat.getInteger(MediaFormat.KEY_BIT_RATE));
outputFormat.setInteger(
MediaFormat.KEY_FRAME_RATE, inputFormat.getInteger(MediaFormat.KEY_FRAME_RATE));
outputFormat.setInteger(
MediaFormat.KEY_I_FRAME_INTERVAL,
inputFormat.getInteger(MediaFormat.KEY_I_FRAME_INTERVAL));
outputData.setMediaFormat(outputFormat);
encoder = MediaCodec.createEncoderByType(MIME_TYPE);
encoder.configure(outputFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
inputSurface = new InputSurface(encoder.createInputSurface());
inputSurface.makeCurrent();
encoder.start();
// OutputSurface uses the EGL context created by InputSurface.
decoder = MediaCodec.createDecoderByType(MIME_TYPE);
outputSurface = new OutputSurface();
outputSurface.changeFragmentShader(FRAGMENT_SHADER);
decoder.configure(inputFormat, outputSurface.getSurface(), null, 0);
decoder.start();
editVideoData(inputData, decoder, outputSurface, inputSurface, encoder, outputData);
} finally {
if (VERBOSE) Log.d(TAG, "shutting down encoder, decoder");
if (outputSurface != null) {
outputSurface.release();
}
if (inputSurface != null) {
inputSurface.release();
}
if (encoder != null) {
encoder.stop();
encoder.release();
}
if (decoder != null) {
decoder.stop();
decoder.release();
}
}
return outputData;
}
/**
* Checks the video file to see if the contents match our expectations. We decode the video to a
* Surface and check the pixels with GL.
*/
private void checkVideoFile(VideoChunks inputData) {
OutputSurface surface = null;
MediaCodec decoder = null;
mLargestColorDelta = -1;
if (VERBOSE) Log.d(TAG, "checkVideoFile");
try {
surface = new OutputSurface(mWidth, mHeight);
MediaFormat format = inputData.getMediaFormat();
decoder = MediaCodec.createDecoderByType(MIME_TYPE);
decoder.configure(format, surface.getSurface(), null, 0);
decoder.start();
int badFrames = checkVideoData(inputData, decoder, surface);
if (badFrames != 0) {
fail("Found " + badFrames + " bad frames");
}
} finally {
if (surface != null) {
surface.release();
}
if (decoder != null) {
decoder.stop();
decoder.release();
}
Log.i(TAG, "Largest color delta: " + mLargestColorDelta);
}
}
private final void handleStop() {
if (DEBUG) Log.v(TAG, "handleStop:");
synchronized (mVideoTask) {
internal_stop_video();
mVideoTrackIndex = -1;
}
if (mVideoMediaCodec != null) {
mVideoMediaCodec.stop();
mVideoMediaCodec.release();
mVideoMediaCodec = null;
}
if (mVideoMediaExtractor != null) {
mVideoMediaExtractor.release();
mVideoMediaExtractor = null;
}
mVideoBufferInfo = null;
mVideoInputBuffers = mVideoOutputBuffers = null;
if (mMetadata != null) {
mMetadata.release();
mMetadata = null;
}
synchronized (mSync) {
mVideoOutputDone = mVideoInputDone = true;
mState = STATE_STOP;
}
mCallback.onFinished();
}
/** Release all releated objects */
protected void release() {
if (DEBUG) Log.d(TAG, "release:");
try {
mListener.onStopped(this);
} catch (Exception e) {
Log.e(TAG, "failed onStopped", e);
}
mIsCapturing = false;
if (mMediaCodec != null) {
try {
mMediaCodec.stop();
mMediaCodec.release();
mMediaCodec = null;
} catch (Exception e) {
Log.e(TAG, "failed releasing MediaCodec", e);
}
}
if (mMuxerStarted) {
final MediaMuxerWrapper muxer = mWeakMuxer.get();
if (muxer != null) {
try {
muxer.stop();
} catch (Exception e) {
Log.e(TAG, "failed stopping muxer", e);
}
}
}
mBufferInfo = null;
}
@Override
public void onDestroy() {
super.onDestroy();
if (encoder != null) {
encoder.stop();
encoder.release();
encoder = null;
}
}
/**
* Generates a test video file, saving it as VideoChunks. We generate frames with GL to avoid
* having to deal with multiple YUV formats.
*
* @return true on success, false on "soft" failure
*/
private boolean generateVideoFile(VideoChunks output) {
if (VERBOSE) Log.d(TAG, "generateVideoFile " + mWidth + "x" + mHeight);
MediaCodec encoder = null;
InputSurface inputSurface = null;
try {
MediaCodecInfo codecInfo = selectCodec(MIME_TYPE);
if (codecInfo == null) {
// Don't fail CTS if they don't have an AVC codec (not here, anyway).
Log.e(TAG, "Unable to find an appropriate codec for " + MIME_TYPE);
return false;
}
if (VERBOSE) Log.d(TAG, "found codec: " + codecInfo.getName());
// We avoid the device-specific limitations on width and height by using values that
// are multiples of 16, which all tested devices seem to be able to handle.
MediaFormat format = MediaFormat.createVideoFormat(MIME_TYPE, mWidth, mHeight);
// Set some properties. Failing to specify some of these can cause the MediaCodec
// configure() call to throw an unhelpful exception.
format.setInteger(
MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
format.setInteger(MediaFormat.KEY_BIT_RATE, mBitRate);
format.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE);
format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL);
if (VERBOSE) Log.d(TAG, "format: " + format);
output.setMediaFormat(format);
// Create a MediaCodec for the desired codec, then configure it as an encoder with
// our desired properties.
encoder = MediaCodec.createByCodecName(codecInfo.getName());
encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
inputSurface = new InputSurface(encoder.createInputSurface());
inputSurface.makeCurrent();
encoder.start();
generateVideoData(encoder, inputSurface, output);
} finally {
if (encoder != null) {
if (VERBOSE) Log.d(TAG, "releasing encoder");
encoder.stop();
encoder.release();
if (VERBOSE) Log.d(TAG, "released encoder");
}
if (inputSurface != null) {
inputSurface.release();
}
}
return true;
}
private void releaseEncoder() {
// Release encoder
if (VERBOSE) {
Log.v(TAG, "releasing encoder");
}
if (mEncoder != null) {
mEncoder.stop();
mEncoder.release();
if (mRecordingSurface != null) {
mRecordingSurface.release();
}
mEncoder = null;
}
}
private void stop() {
if (mThread != null) {
mThread.interrupt();
try {
mThread.join();
} catch (InterruptedException e) {
Log.e(TAG, "Interrupted when waiting for video decoder thread to stop");
}
try {
decoder.stop();
} catch (IllegalStateException e) {
Log.e(TAG, "Video decoder is not in a state that can be stopped");
}
}
mThread = null;
}
private void release() {
if (mEncoder != null) {
mEncoder.stop();
mEncoder.release();
mEncoder = null;
}
if (mVirtualDisplay != null) {
mVirtualDisplay.release();
}
if (displayManager != null) {
displayManager = null;
}
if (mediaMuxer != null) {
mediaMuxer.stop();
mediaMuxer.release();
mediaMuxer = null;
}
}
private void release() {
Logging.d(TAG, "Java releaseDecoder");
checkOnMediaCodecThread();
try {
mediaCodec.stop();
mediaCodec.release();
} catch (IllegalStateException e) {
Logging.e(TAG, "release failed", e);
}
mediaCodec = null;
mediaCodecThread = null;
if (useSurface) {
surface.release();
if (textureID != 0) {
Logging.d(TAG, "Delete video decoder TextureID " + textureID);
GLES20.glDeleteTextures(1, new int[] {textureID}, 0);
textureID = 0;
}
eglBase.release();
eglBase = null;
}
}
@CalledByNative
private static boolean decodeAudioFile(
Context ctx, int nativeMediaCodecBridge, int inputFD, long dataSize) {
if (dataSize < 0 || dataSize > 0x7fffffff) return false;
MediaExtractor extractor = new MediaExtractor();
ParcelFileDescriptor encodedFD;
encodedFD = ParcelFileDescriptor.adoptFd(inputFD);
try {
extractor.setDataSource(encodedFD.getFileDescriptor(), 0, dataSize);
} catch (Exception e) {
e.printStackTrace();
encodedFD.detachFd();
return false;
}
if (extractor.getTrackCount() <= 0) {
encodedFD.detachFd();
return false;
}
MediaFormat format = extractor.getTrackFormat(0);
// Number of channels specified in the file
int inputChannelCount = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
// Number of channels the decoder will provide. (Not
// necessarily the same as inputChannelCount. See
// crbug.com/266006.)
int outputChannelCount = inputChannelCount;
int sampleRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE);
String mime = format.getString(MediaFormat.KEY_MIME);
long durationMicroseconds = 0;
if (format.containsKey(MediaFormat.KEY_DURATION)) {
try {
durationMicroseconds = format.getLong(MediaFormat.KEY_DURATION);
} catch (Exception e) {
Log.d(LOG_TAG, "Cannot get duration");
}
}
if (DEBUG) {
Log.d(
LOG_TAG,
"Tracks: "
+ extractor.getTrackCount()
+ " Rate: "
+ sampleRate
+ " Channels: "
+ inputChannelCount
+ " Mime: "
+ mime
+ " Duration: "
+ durationMicroseconds
+ " microsec");
}
nativeInitializeDestination(
nativeMediaCodecBridge, inputChannelCount, sampleRate, durationMicroseconds);
// Create decoder
MediaCodec codec = MediaCodec.createDecoderByType(mime);
codec.configure(format, null /* surface */, null /* crypto */, 0 /* flags */);
codec.start();
ByteBuffer[] codecInputBuffers = codec.getInputBuffers();
ByteBuffer[] codecOutputBuffers = codec.getOutputBuffers();
// A track must be selected and will be used to read samples.
extractor.selectTrack(0);
boolean sawInputEOS = false;
boolean sawOutputEOS = false;
// Keep processing until the output is done.
while (!sawOutputEOS) {
if (!sawInputEOS) {
// Input side
int inputBufIndex = codec.dequeueInputBuffer(TIMEOUT_MICROSECONDS);
if (inputBufIndex >= 0) {
ByteBuffer dstBuf = codecInputBuffers[inputBufIndex];
int sampleSize = extractor.readSampleData(dstBuf, 0);
long presentationTimeMicroSec = 0;
if (sampleSize < 0) {
sawInputEOS = true;
sampleSize = 0;
} else {
presentationTimeMicroSec = extractor.getSampleTime();
}
codec.queueInputBuffer(
inputBufIndex,
0, /* offset */
sampleSize,
presentationTimeMicroSec,
sawInputEOS ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0);
if (!sawInputEOS) {
extractor.advance();
}
}
}
// Output side
MediaCodec.BufferInfo info = new BufferInfo();
final int outputBufIndex = codec.dequeueOutputBuffer(info, TIMEOUT_MICROSECONDS);
if (outputBufIndex >= 0) {
ByteBuffer buf = codecOutputBuffers[outputBufIndex];
if (info.size > 0) {
nativeOnChunkDecoded(
nativeMediaCodecBridge, buf, info.size, inputChannelCount, outputChannelCount);
}
buf.clear();
codec.releaseOutputBuffer(outputBufIndex, false /* render */);
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
sawOutputEOS = true;
}
} else if (outputBufIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
codecOutputBuffers = codec.getOutputBuffers();
} else if (outputBufIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
MediaFormat newFormat = codec.getOutputFormat();
outputChannelCount = newFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
Log.d(LOG_TAG, "output format changed to " + newFormat);
}
}
encodedFD.detachFd();
codec.stop();
codec.release();
codec = null;
return true;
}
/** Releases resources and ends the encoding/decoding session. */
public void stopAndRelease() {
mDecoder.stop();
mDecoder.release();
mDecoder = null;
mHandler = null;
}
@Override
public void run() {
setup();
synchronized (setupSignal) {
setupSignal.notify();
}
if (!valid) {
return;
}
decoder.start();
ByteBuffer[] inputBuffers = decoder.getInputBuffers();
ByteBuffer[] outputBuffers = decoder.getOutputBuffers();
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
boolean isEOS = false;
long startMs = System.currentTimeMillis();
long timeoutUs = 10000;
int iframe = 0;
while (!Thread.interrupted()) {
if (!isEOS) {
int inIndex = decoder.dequeueInputBuffer(10000);
if (inIndex >= 0) {
ByteBuffer buffer = inputBuffers[inIndex];
int sampleSize = extractor.readSampleData(buffer, 0);
if (sampleSize < 0) {
if (LOCAL_LOGD) {
Log.d("VideoDecoderForOpenCV", "InputBuffer BUFFER_FLAG_END_OF_STREAM");
}
decoder.queueInputBuffer(inIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
isEOS = true;
} else {
decoder.queueInputBuffer(inIndex, 0, sampleSize, extractor.getSampleTime(), 0);
extractor.advance();
}
}
}
int outIndex = decoder.dequeueOutputBuffer(info, 10000);
MediaFormat outFormat;
switch (outIndex) {
case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
if (LOCAL_LOGD) {
Log.d("VideoDecoderForOpenCV", "INFO_OUTPUT_BUFFERS_CHANGED");
}
outputBuffers = decoder.getOutputBuffers();
break;
case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
outFormat = decoder.getOutputFormat();
if (LOCAL_LOGD) {
Log.d("VideoDecoderForOpenCV", "New format " + outFormat);
}
break;
case MediaCodec.INFO_TRY_AGAIN_LATER:
if (LOCAL_LOGD) {
Log.d("VideoDecoderForOpenCV", "dequeueOutputBuffer timed out!");
}
break;
default:
ByteBuffer buffer = outputBuffers[outIndex];
boolean doRender = (info.size != 0);
// As soon as we call releaseOutputBuffer, the buffer will be forwarded
// to SurfaceTexture to convert to a texture. The API doesn't
// guarantee that the texture will be available before the call
// returns, so we need to wait for the onFrameAvailable callback to
// fire. If we don't wait, we risk rendering from the previous frame.
decoder.releaseOutputBuffer(outIndex, doRender);
if (doRender) {
surface.awaitNewImage();
surface.drawImage();
if (LOCAL_LOGD) {
Log.d("VideoDecoderForOpenCV", "Finish drawing a frame!");
}
if ((iframe++ % mDecimation) == 0) {
// Send the frame for processing
mMatBuffer.put();
}
}
break;
}
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
if (LOCAL_LOGD) {
Log.d("VideoDecoderForOpenCV", "OutputBuffer BUFFER_FLAG_END_OF_STREAM");
}
break;
}
}
mMatBuffer.invalidate();
decoder.stop();
teardown();
mThread = null;
}
@Override
public void run() {
isDecoding = true;
codec.start();
@SuppressWarnings("deprecation")
ByteBuffer[] inputBuffers = codec.getInputBuffers();
@SuppressWarnings("deprecation")
ByteBuffer[] outputBuffers = codec.getOutputBuffers();
boolean sawInputEOS = false;
boolean sawOutputEOS = false;
while (!sawInputEOS && !sawOutputEOS && continuing) {
if (state == State.PAUSED) {
try {
synchronized (decoderLock) {
decoderLock.wait();
}
} catch (InterruptedException e) {
// Purposely not doing anything here
}
continue;
}
if (sonic != null) {
sonic.setSpeed(speed);
sonic.setPitch(1);
}
int inputBufIndex = codec.dequeueInputBuffer(200);
if (inputBufIndex >= 0) {
ByteBuffer dstBuf = inputBuffers[inputBufIndex];
int sampleSize = extractor.readSampleData(dstBuf, 0);
long presentationTimeUs = 0;
if (sampleSize < 0) {
sawInputEOS = true;
sampleSize = 0;
} else {
presentationTimeUs = extractor.getSampleTime();
}
codec.queueInputBuffer(
inputBufIndex,
0,
sampleSize,
presentationTimeUs,
sawInputEOS ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0);
if (flushCodec) {
codec.flush();
flushCodec = false;
}
if (!sawInputEOS) {
extractor.advance();
}
}
final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
byte[] modifiedSamples = new byte[info.size];
int res;
//noinspection deprecation
do {
res = codec.dequeueOutputBuffer(info, 200);
if (res >= 0) {
final byte[] chunk = new byte[info.size];
outputBuffers[res].get(chunk);
outputBuffers[res].clear();
if (chunk.length > 0) {
sonic.writeBytesToStream(chunk, chunk.length);
} else {
sonic.flushStream();
}
int available = sonic.availableBytes();
if (available > 0) {
if (modifiedSamples.length < available) {
modifiedSamples = new byte[available];
}
sonic.readBytesFromStream(modifiedSamples, available);
track.write(modifiedSamples, 0, available);
}
codec.releaseOutputBuffer(res, false);
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
sawOutputEOS = true;
}
} else //noinspection deprecation
if (res == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
//noinspection deprecation
outputBuffers = codec.getOutputBuffers();
} else if (res == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
track.stop();
lock.lock();
try {
track.release();
final MediaFormat oFormat = codec.getOutputFormat();
initDevice(
oFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE),
oFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT));
//noinspection deprecation
outputBuffers = codec.getOutputBuffers();
track.play();
} catch (IOException e) {
e.printStackTrace();
} finally {
lock.unlock();
}
}
} while (res == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED
|| res == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED);
}
codec.stop();
track.stop();
isDecoding = false;
if (continuing && (sawInputEOS || sawOutputEOS)) {
state = State.PLAYBACK_COMPLETED;
L.d(TAG, "State changed to: " + state);
Thread t =
new Thread(
new Runnable() {
@Override
public void run() {
if (onCompletionListener != null) {
onCompletionListener.onCompletion();
}
stayAwake(false);
}
});
t.setDaemon(true);
t.start();
}
synchronized (decoderLock) {
decoderLock.notifyAll();
}
}