/** * Edits a video file, saving the contents to a new file. This involves decoding and re-encoding, * not to mention conversions between YUV and RGB, and so may be lossy. * * <p>If we recognize the decoded format we can do this in Java code using the ByteBuffer[] * output, but it's not practical to support all OEM formats. By using a SurfaceTexture for output * and a Surface for input, we can avoid issues with obscure formats and can use a fragment shader * to do transformations. */ private VideoChunks editVideoFile(VideoChunks inputData) { if (VERBOSE) Log.d(TAG, "editVideoFile " + mWidth + "x" + mHeight); VideoChunks outputData = new VideoChunks(); MediaCodec decoder = null; MediaCodec encoder = null; InputSurface inputSurface = null; OutputSurface outputSurface = null; try { MediaFormat inputFormat = inputData.getMediaFormat(); // Create an encoder format that matches the input format. (Might be able to just // re-use the format used to generate the video, since we want it to be the same.) MediaFormat outputFormat = MediaFormat.createVideoFormat(MIME_TYPE, mWidth, mHeight); outputFormat.setInteger( MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); outputFormat.setInteger( MediaFormat.KEY_BIT_RATE, inputFormat.getInteger(MediaFormat.KEY_BIT_RATE)); outputFormat.setInteger( MediaFormat.KEY_FRAME_RATE, inputFormat.getInteger(MediaFormat.KEY_FRAME_RATE)); outputFormat.setInteger( MediaFormat.KEY_I_FRAME_INTERVAL, inputFormat.getInteger(MediaFormat.KEY_I_FRAME_INTERVAL)); outputData.setMediaFormat(outputFormat); encoder = MediaCodec.createEncoderByType(MIME_TYPE); encoder.configure(outputFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); inputSurface = new InputSurface(encoder.createInputSurface()); inputSurface.makeCurrent(); encoder.start(); // OutputSurface uses the EGL context created by InputSurface. decoder = MediaCodec.createDecoderByType(MIME_TYPE); outputSurface = new OutputSurface(); outputSurface.changeFragmentShader(FRAGMENT_SHADER); decoder.configure(inputFormat, outputSurface.getSurface(), null, 0); decoder.start(); editVideoData(inputData, decoder, outputSurface, inputSurface, encoder, outputData); } finally { if (VERBOSE) Log.d(TAG, "shutting down encoder, decoder"); if (outputSurface != null) { outputSurface.release(); } if (inputSurface != null) { inputSurface.release(); } if (encoder != null) { encoder.stop(); encoder.release(); } if (decoder != null) { decoder.stop(); decoder.release(); } } return outputData; }
/** * Checks the video file to see if the contents match our expectations. We decode the video to a * Surface and check the pixels with GL. */ private void checkVideoFile(VideoChunks inputData) { OutputSurface surface = null; MediaCodec decoder = null; mLargestColorDelta = -1; if (VERBOSE) Log.d(TAG, "checkVideoFile"); try { surface = new OutputSurface(mWidth, mHeight); MediaFormat format = inputData.getMediaFormat(); decoder = MediaCodec.createDecoderByType(MIME_TYPE); decoder.configure(format, surface.getSurface(), null, 0); decoder.start(); int badFrames = checkVideoData(inputData, decoder, surface); if (badFrames != 0) { fail("Found " + badFrames + " bad frames"); } } finally { if (surface != null) { surface.release(); } if (decoder != null) { decoder.stop(); decoder.release(); } Log.i(TAG, "Largest color delta: " + mLargestColorDelta); } }
private final void handleStop() { if (DEBUG) Log.v(TAG, "handleStop:"); synchronized (mVideoTask) { internal_stop_video(); mVideoTrackIndex = -1; } if (mVideoMediaCodec != null) { mVideoMediaCodec.stop(); mVideoMediaCodec.release(); mVideoMediaCodec = null; } if (mVideoMediaExtractor != null) { mVideoMediaExtractor.release(); mVideoMediaExtractor = null; } mVideoBufferInfo = null; mVideoInputBuffers = mVideoOutputBuffers = null; if (mMetadata != null) { mMetadata.release(); mMetadata = null; } synchronized (mSync) { mVideoOutputDone = mVideoInputDone = true; mState = STATE_STOP; } mCallback.onFinished(); }
/** Release all releated objects */ protected void release() { if (DEBUG) Log.d(TAG, "release:"); try { mListener.onStopped(this); } catch (Exception e) { Log.e(TAG, "failed onStopped", e); } mIsCapturing = false; if (mMediaCodec != null) { try { mMediaCodec.stop(); mMediaCodec.release(); mMediaCodec = null; } catch (Exception e) { Log.e(TAG, "failed releasing MediaCodec", e); } } if (mMuxerStarted) { final MediaMuxerWrapper muxer = mWeakMuxer.get(); if (muxer != null) { try { muxer.stop(); } catch (Exception e) { Log.e(TAG, "failed stopping muxer", e); } } } mBufferInfo = null; }
@Override public void onDestroy() { super.onDestroy(); if (encoder != null) { encoder.stop(); encoder.release(); encoder = null; } }
/** * Generates a test video file, saving it as VideoChunks. We generate frames with GL to avoid * having to deal with multiple YUV formats. * * @return true on success, false on "soft" failure */ private boolean generateVideoFile(VideoChunks output) { if (VERBOSE) Log.d(TAG, "generateVideoFile " + mWidth + "x" + mHeight); MediaCodec encoder = null; InputSurface inputSurface = null; try { MediaCodecInfo codecInfo = selectCodec(MIME_TYPE); if (codecInfo == null) { // Don't fail CTS if they don't have an AVC codec (not here, anyway). Log.e(TAG, "Unable to find an appropriate codec for " + MIME_TYPE); return false; } if (VERBOSE) Log.d(TAG, "found codec: " + codecInfo.getName()); // We avoid the device-specific limitations on width and height by using values that // are multiples of 16, which all tested devices seem to be able to handle. MediaFormat format = MediaFormat.createVideoFormat(MIME_TYPE, mWidth, mHeight); // Set some properties. Failing to specify some of these can cause the MediaCodec // configure() call to throw an unhelpful exception. format.setInteger( MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); format.setInteger(MediaFormat.KEY_BIT_RATE, mBitRate); format.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE); format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL); if (VERBOSE) Log.d(TAG, "format: " + format); output.setMediaFormat(format); // Create a MediaCodec for the desired codec, then configure it as an encoder with // our desired properties. encoder = MediaCodec.createByCodecName(codecInfo.getName()); encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); inputSurface = new InputSurface(encoder.createInputSurface()); inputSurface.makeCurrent(); encoder.start(); generateVideoData(encoder, inputSurface, output); } finally { if (encoder != null) { if (VERBOSE) Log.d(TAG, "releasing encoder"); encoder.stop(); encoder.release(); if (VERBOSE) Log.d(TAG, "released encoder"); } if (inputSurface != null) { inputSurface.release(); } } return true; }
private void releaseEncoder() { // Release encoder if (VERBOSE) { Log.v(TAG, "releasing encoder"); } if (mEncoder != null) { mEncoder.stop(); mEncoder.release(); if (mRecordingSurface != null) { mRecordingSurface.release(); } mEncoder = null; } }
private void stop() { if (mThread != null) { mThread.interrupt(); try { mThread.join(); } catch (InterruptedException e) { Log.e(TAG, "Interrupted when waiting for video decoder thread to stop"); } try { decoder.stop(); } catch (IllegalStateException e) { Log.e(TAG, "Video decoder is not in a state that can be stopped"); } } mThread = null; }
private void release() { if (mEncoder != null) { mEncoder.stop(); mEncoder.release(); mEncoder = null; } if (mVirtualDisplay != null) { mVirtualDisplay.release(); } if (displayManager != null) { displayManager = null; } if (mediaMuxer != null) { mediaMuxer.stop(); mediaMuxer.release(); mediaMuxer = null; } }
private void release() { Logging.d(TAG, "Java releaseDecoder"); checkOnMediaCodecThread(); try { mediaCodec.stop(); mediaCodec.release(); } catch (IllegalStateException e) { Logging.e(TAG, "release failed", e); } mediaCodec = null; mediaCodecThread = null; if (useSurface) { surface.release(); if (textureID != 0) { Logging.d(TAG, "Delete video decoder TextureID " + textureID); GLES20.glDeleteTextures(1, new int[] {textureID}, 0); textureID = 0; } eglBase.release(); eglBase = null; } }
@CalledByNative private static boolean decodeAudioFile( Context ctx, int nativeMediaCodecBridge, int inputFD, long dataSize) { if (dataSize < 0 || dataSize > 0x7fffffff) return false; MediaExtractor extractor = new MediaExtractor(); ParcelFileDescriptor encodedFD; encodedFD = ParcelFileDescriptor.adoptFd(inputFD); try { extractor.setDataSource(encodedFD.getFileDescriptor(), 0, dataSize); } catch (Exception e) { e.printStackTrace(); encodedFD.detachFd(); return false; } if (extractor.getTrackCount() <= 0) { encodedFD.detachFd(); return false; } MediaFormat format = extractor.getTrackFormat(0); // Number of channels specified in the file int inputChannelCount = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT); // Number of channels the decoder will provide. (Not // necessarily the same as inputChannelCount. See // crbug.com/266006.) int outputChannelCount = inputChannelCount; int sampleRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE); String mime = format.getString(MediaFormat.KEY_MIME); long durationMicroseconds = 0; if (format.containsKey(MediaFormat.KEY_DURATION)) { try { durationMicroseconds = format.getLong(MediaFormat.KEY_DURATION); } catch (Exception e) { Log.d(LOG_TAG, "Cannot get duration"); } } if (DEBUG) { Log.d( LOG_TAG, "Tracks: " + extractor.getTrackCount() + " Rate: " + sampleRate + " Channels: " + inputChannelCount + " Mime: " + mime + " Duration: " + durationMicroseconds + " microsec"); } nativeInitializeDestination( nativeMediaCodecBridge, inputChannelCount, sampleRate, durationMicroseconds); // Create decoder MediaCodec codec = MediaCodec.createDecoderByType(mime); codec.configure(format, null /* surface */, null /* crypto */, 0 /* flags */); codec.start(); ByteBuffer[] codecInputBuffers = codec.getInputBuffers(); ByteBuffer[] codecOutputBuffers = codec.getOutputBuffers(); // A track must be selected and will be used to read samples. extractor.selectTrack(0); boolean sawInputEOS = false; boolean sawOutputEOS = false; // Keep processing until the output is done. while (!sawOutputEOS) { if (!sawInputEOS) { // Input side int inputBufIndex = codec.dequeueInputBuffer(TIMEOUT_MICROSECONDS); if (inputBufIndex >= 0) { ByteBuffer dstBuf = codecInputBuffers[inputBufIndex]; int sampleSize = extractor.readSampleData(dstBuf, 0); long presentationTimeMicroSec = 0; if (sampleSize < 0) { sawInputEOS = true; sampleSize = 0; } else { presentationTimeMicroSec = extractor.getSampleTime(); } codec.queueInputBuffer( inputBufIndex, 0, /* offset */ sampleSize, presentationTimeMicroSec, sawInputEOS ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0); if (!sawInputEOS) { extractor.advance(); } } } // Output side MediaCodec.BufferInfo info = new BufferInfo(); final int outputBufIndex = codec.dequeueOutputBuffer(info, TIMEOUT_MICROSECONDS); if (outputBufIndex >= 0) { ByteBuffer buf = codecOutputBuffers[outputBufIndex]; if (info.size > 0) { nativeOnChunkDecoded( nativeMediaCodecBridge, buf, info.size, inputChannelCount, outputChannelCount); } buf.clear(); codec.releaseOutputBuffer(outputBufIndex, false /* render */); if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { sawOutputEOS = true; } } else if (outputBufIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { codecOutputBuffers = codec.getOutputBuffers(); } else if (outputBufIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { MediaFormat newFormat = codec.getOutputFormat(); outputChannelCount = newFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); Log.d(LOG_TAG, "output format changed to " + newFormat); } } encodedFD.detachFd(); codec.stop(); codec.release(); codec = null; return true; }
/** Releases resources and ends the encoding/decoding session. */ public void stopAndRelease() { mDecoder.stop(); mDecoder.release(); mDecoder = null; mHandler = null; }
@Override public void run() { setup(); synchronized (setupSignal) { setupSignal.notify(); } if (!valid) { return; } decoder.start(); ByteBuffer[] inputBuffers = decoder.getInputBuffers(); ByteBuffer[] outputBuffers = decoder.getOutputBuffers(); MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); boolean isEOS = false; long startMs = System.currentTimeMillis(); long timeoutUs = 10000; int iframe = 0; while (!Thread.interrupted()) { if (!isEOS) { int inIndex = decoder.dequeueInputBuffer(10000); if (inIndex >= 0) { ByteBuffer buffer = inputBuffers[inIndex]; int sampleSize = extractor.readSampleData(buffer, 0); if (sampleSize < 0) { if (LOCAL_LOGD) { Log.d("VideoDecoderForOpenCV", "InputBuffer BUFFER_FLAG_END_OF_STREAM"); } decoder.queueInputBuffer(inIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM); isEOS = true; } else { decoder.queueInputBuffer(inIndex, 0, sampleSize, extractor.getSampleTime(), 0); extractor.advance(); } } } int outIndex = decoder.dequeueOutputBuffer(info, 10000); MediaFormat outFormat; switch (outIndex) { case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: if (LOCAL_LOGD) { Log.d("VideoDecoderForOpenCV", "INFO_OUTPUT_BUFFERS_CHANGED"); } outputBuffers = decoder.getOutputBuffers(); break; case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: outFormat = decoder.getOutputFormat(); if (LOCAL_LOGD) { Log.d("VideoDecoderForOpenCV", "New format " + outFormat); } break; case MediaCodec.INFO_TRY_AGAIN_LATER: if (LOCAL_LOGD) { Log.d("VideoDecoderForOpenCV", "dequeueOutputBuffer timed out!"); } break; default: ByteBuffer buffer = outputBuffers[outIndex]; boolean doRender = (info.size != 0); // As soon as we call releaseOutputBuffer, the buffer will be forwarded // to SurfaceTexture to convert to a texture. The API doesn't // guarantee that the texture will be available before the call // returns, so we need to wait for the onFrameAvailable callback to // fire. If we don't wait, we risk rendering from the previous frame. decoder.releaseOutputBuffer(outIndex, doRender); if (doRender) { surface.awaitNewImage(); surface.drawImage(); if (LOCAL_LOGD) { Log.d("VideoDecoderForOpenCV", "Finish drawing a frame!"); } if ((iframe++ % mDecimation) == 0) { // Send the frame for processing mMatBuffer.put(); } } break; } if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { if (LOCAL_LOGD) { Log.d("VideoDecoderForOpenCV", "OutputBuffer BUFFER_FLAG_END_OF_STREAM"); } break; } } mMatBuffer.invalidate(); decoder.stop(); teardown(); mThread = null; }
@Override public void run() { isDecoding = true; codec.start(); @SuppressWarnings("deprecation") ByteBuffer[] inputBuffers = codec.getInputBuffers(); @SuppressWarnings("deprecation") ByteBuffer[] outputBuffers = codec.getOutputBuffers(); boolean sawInputEOS = false; boolean sawOutputEOS = false; while (!sawInputEOS && !sawOutputEOS && continuing) { if (state == State.PAUSED) { try { synchronized (decoderLock) { decoderLock.wait(); } } catch (InterruptedException e) { // Purposely not doing anything here } continue; } if (sonic != null) { sonic.setSpeed(speed); sonic.setPitch(1); } int inputBufIndex = codec.dequeueInputBuffer(200); if (inputBufIndex >= 0) { ByteBuffer dstBuf = inputBuffers[inputBufIndex]; int sampleSize = extractor.readSampleData(dstBuf, 0); long presentationTimeUs = 0; if (sampleSize < 0) { sawInputEOS = true; sampleSize = 0; } else { presentationTimeUs = extractor.getSampleTime(); } codec.queueInputBuffer( inputBufIndex, 0, sampleSize, presentationTimeUs, sawInputEOS ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0); if (flushCodec) { codec.flush(); flushCodec = false; } if (!sawInputEOS) { extractor.advance(); } } final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); byte[] modifiedSamples = new byte[info.size]; int res; //noinspection deprecation do { res = codec.dequeueOutputBuffer(info, 200); if (res >= 0) { final byte[] chunk = new byte[info.size]; outputBuffers[res].get(chunk); outputBuffers[res].clear(); if (chunk.length > 0) { sonic.writeBytesToStream(chunk, chunk.length); } else { sonic.flushStream(); } int available = sonic.availableBytes(); if (available > 0) { if (modifiedSamples.length < available) { modifiedSamples = new byte[available]; } sonic.readBytesFromStream(modifiedSamples, available); track.write(modifiedSamples, 0, available); } codec.releaseOutputBuffer(res, false); if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { sawOutputEOS = true; } } else //noinspection deprecation if (res == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { //noinspection deprecation outputBuffers = codec.getOutputBuffers(); } else if (res == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { track.stop(); lock.lock(); try { track.release(); final MediaFormat oFormat = codec.getOutputFormat(); initDevice( oFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE), oFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT)); //noinspection deprecation outputBuffers = codec.getOutputBuffers(); track.play(); } catch (IOException e) { e.printStackTrace(); } finally { lock.unlock(); } } } while (res == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED || res == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED); } codec.stop(); track.stop(); isDecoding = false; if (continuing && (sawInputEOS || sawOutputEOS)) { state = State.PLAYBACK_COMPLETED; L.d(TAG, "State changed to: " + state); Thread t = new Thread( new Runnable() { @Override public void run() { if (onCompletionListener != null) { onCompletionListener.onCompletion(); } stayAwake(false); } }); t.setDaemon(true); t.start(); } synchronized (decoderLock) { decoderLock.notifyAll(); } }