/** * Edits a video file, saving the contents to a new file. This involves decoding and re-encoding, * not to mention conversions between YUV and RGB, and so may be lossy. * * <p>If we recognize the decoded format we can do this in Java code using the ByteBuffer[] * output, but it's not practical to support all OEM formats. By using a SurfaceTexture for output * and a Surface for input, we can avoid issues with obscure formats and can use a fragment shader * to do transformations. */ private VideoChunks editVideoFile(VideoChunks inputData) { if (VERBOSE) Log.d(TAG, "editVideoFile " + mWidth + "x" + mHeight); VideoChunks outputData = new VideoChunks(); MediaCodec decoder = null; MediaCodec encoder = null; InputSurface inputSurface = null; OutputSurface outputSurface = null; try { MediaFormat inputFormat = inputData.getMediaFormat(); // Create an encoder format that matches the input format. (Might be able to just // re-use the format used to generate the video, since we want it to be the same.) MediaFormat outputFormat = MediaFormat.createVideoFormat(MIME_TYPE, mWidth, mHeight); outputFormat.setInteger( MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); outputFormat.setInteger( MediaFormat.KEY_BIT_RATE, inputFormat.getInteger(MediaFormat.KEY_BIT_RATE)); outputFormat.setInteger( MediaFormat.KEY_FRAME_RATE, inputFormat.getInteger(MediaFormat.KEY_FRAME_RATE)); outputFormat.setInteger( MediaFormat.KEY_I_FRAME_INTERVAL, inputFormat.getInteger(MediaFormat.KEY_I_FRAME_INTERVAL)); outputData.setMediaFormat(outputFormat); encoder = MediaCodec.createEncoderByType(MIME_TYPE); encoder.configure(outputFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); inputSurface = new InputSurface(encoder.createInputSurface()); inputSurface.makeCurrent(); encoder.start(); // OutputSurface uses the EGL context created by InputSurface. decoder = MediaCodec.createDecoderByType(MIME_TYPE); outputSurface = new OutputSurface(); outputSurface.changeFragmentShader(FRAGMENT_SHADER); decoder.configure(inputFormat, outputSurface.getSurface(), null, 0); decoder.start(); editVideoData(inputData, decoder, outputSurface, inputSurface, encoder, outputData); } finally { if (VERBOSE) Log.d(TAG, "shutting down encoder, decoder"); if (outputSurface != null) { outputSurface.release(); } if (inputSurface != null) { inputSurface.release(); } if (encoder != null) { encoder.stop(); encoder.release(); } if (decoder != null) { decoder.stop(); decoder.release(); } } return outputData; }
/** * Generates a test video file, saving it as VideoChunks. We generate frames with GL to avoid * having to deal with multiple YUV formats. * * @return true on success, false on "soft" failure */ private boolean generateVideoFile(VideoChunks output) { if (VERBOSE) Log.d(TAG, "generateVideoFile " + mWidth + "x" + mHeight); MediaCodec encoder = null; InputSurface inputSurface = null; try { MediaCodecInfo codecInfo = selectCodec(MIME_TYPE); if (codecInfo == null) { // Don't fail CTS if they don't have an AVC codec (not here, anyway). Log.e(TAG, "Unable to find an appropriate codec for " + MIME_TYPE); return false; } if (VERBOSE) Log.d(TAG, "found codec: " + codecInfo.getName()); // We avoid the device-specific limitations on width and height by using values that // are multiples of 16, which all tested devices seem to be able to handle. MediaFormat format = MediaFormat.createVideoFormat(MIME_TYPE, mWidth, mHeight); // Set some properties. Failing to specify some of these can cause the MediaCodec // configure() call to throw an unhelpful exception. format.setInteger( MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); format.setInteger(MediaFormat.KEY_BIT_RATE, mBitRate); format.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE); format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL); if (VERBOSE) Log.d(TAG, "format: " + format); output.setMediaFormat(format); // Create a MediaCodec for the desired codec, then configure it as an encoder with // our desired properties. encoder = MediaCodec.createByCodecName(codecInfo.getName()); encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); inputSurface = new InputSurface(encoder.createInputSurface()); inputSurface.makeCurrent(); encoder.start(); generateVideoData(encoder, inputSurface, output); } finally { if (encoder != null) { if (VERBOSE) Log.d(TAG, "releasing encoder"); encoder.stop(); encoder.release(); if (VERBOSE) Log.d(TAG, "released encoder"); } if (inputSurface != null) { inputSurface.release(); } } return true; }
/** Edits a stream of video data. */ private void editVideoData( VideoChunks inputData, MediaCodec decoder, OutputSurface outputSurface, InputSurface inputSurface, MediaCodec encoder, VideoChunks outputData) { final int TIMEOUT_USEC = 10000; ByteBuffer[] decoderInputBuffers = decoder.getInputBuffers(); ByteBuffer[] encoderOutputBuffers = encoder.getOutputBuffers(); MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); int inputChunk = 0; int outputCount = 0; boolean outputDone = false; boolean inputDone = false; boolean decoderDone = false; while (!outputDone) { if (VERBOSE) Log.d(TAG, "edit loop"); // Feed more data to the decoder. if (!inputDone) { int inputBufIndex = decoder.dequeueInputBuffer(TIMEOUT_USEC); if (inputBufIndex >= 0) { if (inputChunk == inputData.getNumChunks()) { // End of stream -- send empty frame with EOS flag set. decoder.queueInputBuffer(inputBufIndex, 0, 0, 0L, MediaCodec.BUFFER_FLAG_END_OF_STREAM); inputDone = true; if (VERBOSE) Log.d(TAG, "sent input EOS (with zero-length frame)"); } else { // Copy a chunk of input to the decoder. The first chunk should have // the BUFFER_FLAG_CODEC_CONFIG flag set. ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex]; inputBuf.clear(); inputData.getChunkData(inputChunk, inputBuf); int flags = inputData.getChunkFlags(inputChunk); long time = inputData.getChunkTime(inputChunk); decoder.queueInputBuffer(inputBufIndex, 0, inputBuf.position(), time, flags); if (VERBOSE) { Log.d( TAG, "submitted frame " + inputChunk + " to dec, size=" + inputBuf.position() + " flags=" + flags); } inputChunk++; } } else { if (VERBOSE) Log.d(TAG, "input buffer not available"); } } // Assume output is available. Loop until both assumptions are false. boolean decoderOutputAvailable = !decoderDone; boolean encoderOutputAvailable = true; while (decoderOutputAvailable || encoderOutputAvailable) { // Start by draining any pending output from the encoder. It's important to // do this before we try to stuff any more data in. int encoderStatus = encoder.dequeueOutputBuffer(info, TIMEOUT_USEC); if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) { // no output available yet if (VERBOSE) Log.d(TAG, "no output from encoder available"); encoderOutputAvailable = false; } else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { encoderOutputBuffers = encoder.getOutputBuffers(); if (VERBOSE) Log.d(TAG, "encoder output buffers changed"); } else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { MediaFormat newFormat = encoder.getOutputFormat(); if (VERBOSE) Log.d(TAG, "encoder output format changed: " + newFormat); } else if (encoderStatus < 0) { fail("unexpected result from encoder.dequeueOutputBuffer: " + encoderStatus); } else { // encoderStatus >= 0 ByteBuffer encodedData = encoderOutputBuffers[encoderStatus]; if (encodedData == null) { fail("encoderOutputBuffer " + encoderStatus + " was null"); } // Write the data to the output "file". if (info.size != 0) { encodedData.position(info.offset); encodedData.limit(info.offset + info.size); outputData.addChunk(encodedData, info.flags, info.presentationTimeUs); outputCount++; if (VERBOSE) Log.d(TAG, "encoder output " + info.size + " bytes"); } outputDone = (info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0; encoder.releaseOutputBuffer(encoderStatus, false); } if (encoderStatus != MediaCodec.INFO_TRY_AGAIN_LATER) { // Continue attempts to drain output. continue; } // Encoder is drained, check to see if we've got a new frame of output from // the decoder. (The output is going to a Surface, rather than a ByteBuffer, // but we still get information through BufferInfo.) if (!decoderDone) { int decoderStatus = decoder.dequeueOutputBuffer(info, TIMEOUT_USEC); if (decoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) { // no output available yet if (VERBOSE) Log.d(TAG, "no output from decoder available"); decoderOutputAvailable = false; } else if (decoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { // decoderOutputBuffers = decoder.getOutputBuffers(); if (VERBOSE) Log.d(TAG, "decoder output buffers changed (we don't care)"); } else if (decoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { // expected before first buffer of data MediaFormat newFormat = decoder.getOutputFormat(); if (VERBOSE) Log.d(TAG, "decoder output format changed: " + newFormat); } else if (decoderStatus < 0) { fail("unexpected result from decoder.dequeueOutputBuffer: " + decoderStatus); } else { // decoderStatus >= 0 if (VERBOSE) Log.d( TAG, "surface decoder given buffer " + decoderStatus + " (size=" + info.size + ")"); // The ByteBuffers are null references, but we still get a nonzero // size for the decoded data. boolean doRender = (info.size != 0); // As soon as we call releaseOutputBuffer, the buffer will be forwarded // to SurfaceTexture to convert to a texture. The API doesn't // guarantee that the texture will be available before the call // returns, so we need to wait for the onFrameAvailable callback to // fire. If we don't wait, we risk rendering from the previous frame. decoder.releaseOutputBuffer(decoderStatus, doRender); if (doRender) { // This waits for the image and renders it after it arrives. if (VERBOSE) Log.d(TAG, "awaiting frame"); outputSurface.awaitNewImage(); outputSurface.drawImage(); // Send it to the encoder. inputSurface.setPresentationTime(info.presentationTimeUs * 1000); if (VERBOSE) Log.d(TAG, "swapBuffers"); inputSurface.swapBuffers(); } if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { // forward decoder EOS to encoder if (VERBOSE) Log.d(TAG, "signaling input EOS"); if (WORK_AROUND_BUGS) { // Bail early, possibly dropping a frame. return; } else { encoder.signalEndOfInputStream(); } } } } } } if (inputChunk != outputCount) { throw new RuntimeException("frame lost: " + inputChunk + " in, " + outputCount + " out"); } }
/** * Generates video frames, feeds them into the encoder, and writes the output to the VideoChunks * instance. */ private void generateVideoData( MediaCodec encoder, InputSurface inputSurface, VideoChunks output) { final int TIMEOUT_USEC = 10000; ByteBuffer[] encoderOutputBuffers = encoder.getOutputBuffers(); MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); int generateIndex = 0; int outputCount = 0; // Loop until the output side is done. boolean inputDone = false; boolean outputDone = false; while (!outputDone) { if (VERBOSE) Log.d(TAG, "gen loop"); // If we're not done submitting frames, generate a new one and submit it. The // eglSwapBuffers call will block if the input is full. if (!inputDone) { if (generateIndex == NUM_FRAMES) { // Send an empty frame with the end-of-stream flag set. if (VERBOSE) Log.d(TAG, "signaling input EOS"); if (WORK_AROUND_BUGS) { // Might drop a frame, but at least we won't crash mediaserver. try { Thread.sleep(500); } catch (InterruptedException ie) { } outputDone = true; } else { encoder.signalEndOfInputStream(); } inputDone = true; } else { generateSurfaceFrame(generateIndex); inputSurface.setPresentationTime(computePresentationTime(generateIndex) * 1000); if (VERBOSE) Log.d(TAG, "inputSurface swapBuffers"); inputSurface.swapBuffers(); } generateIndex++; } // Check for output from the encoder. If there's no output yet, we either need to // provide more input, or we need to wait for the encoder to work its magic. We // can't actually tell which is the case, so if we can't get an output buffer right // away we loop around and see if it wants more input. // // If we do find output, drain it all before supplying more input. while (true) { int encoderStatus = encoder.dequeueOutputBuffer(info, TIMEOUT_USEC); if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) { // no output available yet if (VERBOSE) Log.d(TAG, "no output from encoder available"); break; // out of while } else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { // not expected for an encoder encoderOutputBuffers = encoder.getOutputBuffers(); if (VERBOSE) Log.d(TAG, "encoder output buffers changed"); } else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { // not expected for an encoder MediaFormat newFormat = encoder.getOutputFormat(); if (VERBOSE) Log.d(TAG, "encoder output format changed: " + newFormat); } else if (encoderStatus < 0) { fail("unexpected result from encoder.dequeueOutputBuffer: " + encoderStatus); } else { // encoderStatus >= 0 ByteBuffer encodedData = encoderOutputBuffers[encoderStatus]; if (encodedData == null) { fail("encoderOutputBuffer " + encoderStatus + " was null"); } // Codec config flag must be set iff this is the first chunk of output. This // may not hold for all codecs, but it appears to be the case for video/avc. assertTrue((info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0 || outputCount != 0); if (info.size != 0) { // Adjust the ByteBuffer values to match BufferInfo. encodedData.position(info.offset); encodedData.limit(info.offset + info.size); output.addChunk(encodedData, info.flags, info.presentationTimeUs); outputCount++; } encoder.releaseOutputBuffer(encoderStatus, false); if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { outputDone = true; break; // out of while } } } } // One chunk per frame, plus one for the config data. assertEquals("Frame count", NUM_FRAMES + 1, outputCount); }