/**
* Edits a video file, saving the contents to a new file. This involves decoding and re-encoding,
* not to mention conversions between YUV and RGB, and so may be lossy.
*
* <p>If we recognize the decoded format we can do this in Java code using the ByteBuffer[]
* output, but it's not practical to support all OEM formats. By using a SurfaceTexture for output
* and a Surface for input, we can avoid issues with obscure formats and can use a fragment shader
* to do transformations.
*/
private VideoChunks editVideoFile(VideoChunks inputData) {
if (VERBOSE) Log.d(TAG, "editVideoFile " + mWidth + "x" + mHeight);
VideoChunks outputData = new VideoChunks();
MediaCodec decoder = null;
MediaCodec encoder = null;
InputSurface inputSurface = null;
OutputSurface outputSurface = null;
try {
MediaFormat inputFormat = inputData.getMediaFormat();
// Create an encoder format that matches the input format. (Might be able to just
// re-use the format used to generate the video, since we want it to be the same.)
MediaFormat outputFormat = MediaFormat.createVideoFormat(MIME_TYPE, mWidth, mHeight);
outputFormat.setInteger(
MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
outputFormat.setInteger(
MediaFormat.KEY_BIT_RATE, inputFormat.getInteger(MediaFormat.KEY_BIT_RATE));
outputFormat.setInteger(
MediaFormat.KEY_FRAME_RATE, inputFormat.getInteger(MediaFormat.KEY_FRAME_RATE));
outputFormat.setInteger(
MediaFormat.KEY_I_FRAME_INTERVAL,
inputFormat.getInteger(MediaFormat.KEY_I_FRAME_INTERVAL));
outputData.setMediaFormat(outputFormat);
encoder = MediaCodec.createEncoderByType(MIME_TYPE);
encoder.configure(outputFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
inputSurface = new InputSurface(encoder.createInputSurface());
inputSurface.makeCurrent();
encoder.start();
// OutputSurface uses the EGL context created by InputSurface.
decoder = MediaCodec.createDecoderByType(MIME_TYPE);
outputSurface = new OutputSurface();
outputSurface.changeFragmentShader(FRAGMENT_SHADER);
decoder.configure(inputFormat, outputSurface.getSurface(), null, 0);
decoder.start();
editVideoData(inputData, decoder, outputSurface, inputSurface, encoder, outputData);
} finally {
if (VERBOSE) Log.d(TAG, "shutting down encoder, decoder");
if (outputSurface != null) {
outputSurface.release();
}
if (inputSurface != null) {
inputSurface.release();
}
if (encoder != null) {
encoder.stop();
encoder.release();
}
if (decoder != null) {
decoder.stop();
decoder.release();
}
}
return outputData;
}
/**
* Checks the video file to see if the contents match our expectations. We decode the video to a
* Surface and check the pixels with GL.
*/
private void checkVideoFile(VideoChunks inputData) {
OutputSurface surface = null;
MediaCodec decoder = null;
mLargestColorDelta = -1;
if (VERBOSE) Log.d(TAG, "checkVideoFile");
try {
surface = new OutputSurface(mWidth, mHeight);
MediaFormat format = inputData.getMediaFormat();
decoder = MediaCodec.createDecoderByType(MIME_TYPE);
decoder.configure(format, surface.getSurface(), null, 0);
decoder.start();
int badFrames = checkVideoData(inputData, decoder, surface);
if (badFrames != 0) {
fail("Found " + badFrames + " bad frames");
}
} finally {
if (surface != null) {
surface.release();
}
if (decoder != null) {
decoder.stop();
decoder.release();
}
Log.i(TAG, "Largest color delta: " + mLargestColorDelta);
}
}
/**
* Constructs the {@link MediaCodecWrapper} wrapper object around the video codec. The codec is
* created using the encapsulated information in the {@link MediaFormat} object.
*
* @param trackFormat The format of the media object to be decoded.
* @param surface Surface to render the decoded frames.
* @return
*/
public static MediaCodecWrapper fromVideoFormat(final MediaFormat trackFormat, Surface surface) {
MediaCodecWrapper result = null;
MediaCodec videoCodec = null;
// BEGIN_INCLUDE(create_codec)
final String mimeType = trackFormat.getString(MediaFormat.KEY_MIME);
// Check to see if this is actually a video mime type. If it is, then create
// a codec that can decode this mime type.
if (mimeType.contains("video/")) {
try {
videoCodec = MediaCodec.createDecoderByType(mimeType);
videoCodec.configure(trackFormat, surface, null, 0);
} catch (Exception e) {
}
}
// If codec creation was successful, then create a wrapper object around the
// newly created codec.
if (videoCodec != null) {
result = new MediaCodecWrapper(videoCodec);
}
// END_INCLUDE(create_codec)
return result;
}
// Override configureCodec to provide the surface.
@Override
protected void configureCodec(
MediaCodec codec,
boolean codecIsAdaptive,
android.media.MediaFormat format,
MediaCrypto crypto) {
maybeSetMaxInputSize(format, codecIsAdaptive);
codec.configure(format, surface, crypto, 0);
}
/**
* Generates a test video file, saving it as VideoChunks. We generate frames with GL to avoid
* having to deal with multiple YUV formats.
*
* @return true on success, false on "soft" failure
*/
private boolean generateVideoFile(VideoChunks output) {
if (VERBOSE) Log.d(TAG, "generateVideoFile " + mWidth + "x" + mHeight);
MediaCodec encoder = null;
InputSurface inputSurface = null;
try {
MediaCodecInfo codecInfo = selectCodec(MIME_TYPE);
if (codecInfo == null) {
// Don't fail CTS if they don't have an AVC codec (not here, anyway).
Log.e(TAG, "Unable to find an appropriate codec for " + MIME_TYPE);
return false;
}
if (VERBOSE) Log.d(TAG, "found codec: " + codecInfo.getName());
// We avoid the device-specific limitations on width and height by using values that
// are multiples of 16, which all tested devices seem to be able to handle.
MediaFormat format = MediaFormat.createVideoFormat(MIME_TYPE, mWidth, mHeight);
// Set some properties. Failing to specify some of these can cause the MediaCodec
// configure() call to throw an unhelpful exception.
format.setInteger(
MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
format.setInteger(MediaFormat.KEY_BIT_RATE, mBitRate);
format.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE);
format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL);
if (VERBOSE) Log.d(TAG, "format: " + format);
output.setMediaFormat(format);
// Create a MediaCodec for the desired codec, then configure it as an encoder with
// our desired properties.
encoder = MediaCodec.createByCodecName(codecInfo.getName());
encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
inputSurface = new InputSurface(encoder.createInputSurface());
inputSurface.makeCurrent();
encoder.start();
generateVideoData(encoder, inputSurface, output);
} finally {
if (encoder != null) {
if (VERBOSE) Log.d(TAG, "releasing encoder");
encoder.stop();
encoder.release();
if (VERBOSE) Log.d(TAG, "released encoder");
}
if (inputSurface != null) {
inputSurface.release();
}
}
return true;
}
/**
* @param media_extractor
* @param trackIndex
* @return
*/
protected MediaCodec internal_start_video(
final MediaExtractor media_extractor, final int trackIndex) {
if (DEBUG) Log.v(TAG, "internal_start_video:");
MediaCodec codec = null;
if (trackIndex >= 0) {
final MediaFormat format = media_extractor.getTrackFormat(trackIndex);
final String mime = format.getString(MediaFormat.KEY_MIME);
codec = MediaCodec.createDecoderByType(mime);
codec.configure(format, mOutputSurface, null, 0);
codec.start();
if (DEBUG) Log.v(TAG, "internal_start_video:codec started");
}
return codec;
}
private void initStream() throws IOException, IllegalArgumentException {
L.v(TAG, "initStream called in state=" + state);
lock.lock();
try {
extractor = new MediaExtractor();
if (path != null) {
extractor.setDataSource(path);
} else {
error("initStream");
throw new IOException();
}
int trackNum = 0;
final MediaFormat oFormat = extractor.getTrackFormat(trackNum);
if (!oFormat.containsKey(MediaFormat.KEY_SAMPLE_RATE)) {
error("initStream");
throw new IOException("No KEY_SAMPLE_RATE");
}
int sampleRate = oFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE);
if (!oFormat.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) {
error("initStream");
throw new IOException("No KEY_CHANNEL_COUNT");
}
int channelCount = oFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
if (!oFormat.containsKey(MediaFormat.KEY_MIME)) {
error("initStream");
throw new IOException("No KEY_MIME");
}
final String mime = oFormat.getString(MediaFormat.KEY_MIME);
if (!oFormat.containsKey(MediaFormat.KEY_DURATION)) {
error("initStream");
throw new IOException("No KEY_DURATION");
}
duration = oFormat.getLong(MediaFormat.KEY_DURATION);
L.v(TAG, "Sample rate: " + sampleRate);
L.v(TAG, "Mime type: " + mime);
initDevice(sampleRate, channelCount);
extractor.selectTrack(trackNum);
codec = MediaCodec.createDecoderByType(mime);
codec.configure(oFormat, null, null, 0);
} finally {
lock.unlock();
}
}
/**
* Create the display surface out of the encoder. The data to encoder will be fed from this
* Surface itself.
*
* @return
* @throws IOException
*/
@TargetApi(19)
private Surface createDisplaySurface() throws IOException {
MediaFormat mMediaFormat =
MediaFormat.createVideoFormat(CodecUtils.MIME_TYPE, CodecUtils.WIDTH, CodecUtils.HEIGHT);
mMediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, (int) (1024 * 1024 * 0.5));
mMediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 30);
mMediaFormat.setInteger(
MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
mMediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1);
Log.i(TAG, "Starting encoder");
encoder = MediaCodec.createEncoderByType(CodecUtils.MIME_TYPE);
encoder.configure(mMediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
Surface surface = encoder.createInputSurface();
return surface;
}
public void prepareEncode() {
MediaFormat mediaFormat = MediaFormat.createVideoFormat(MIME_TYPE, mWidth, mHeight);
// 描述视频格式的信息
mediaFormat.setInteger(
MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, BITRATE); // 比特率
mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE); // 帧数
mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL); // 关键帧间隔时间
try {
mEncoder = MediaCodec.createEncoderByType(MIME_TYPE);
mEncoder.configure(mediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
mSurface = mEncoder.createInputSurface();
mEncoder.start();
mediaMuxer = new MediaMuxer(filePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
} catch (IOException e) {
e.printStackTrace();
}
}
void setup() {
int width = 0, height = 0;
extractor = new MediaExtractor();
try {
extractor.setDataSource(mVideoFile.getPath());
} catch (IOException e) {
return;
}
for (int i = 0; i < extractor.getTrackCount(); i++) {
MediaFormat format = extractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
width = format.getInteger(MediaFormat.KEY_WIDTH);
height = format.getInteger(MediaFormat.KEY_HEIGHT);
if (mime.startsWith("video/")) {
extractor.selectTrack(i);
try {
decoder = MediaCodec.createDecoderByType(mime);
} catch (IOException e) {
continue;
}
// Decode to surface
// decoder.configure(format, surface, null, 0);
// Decode to offscreen surface
surface = new CtsMediaOutputSurface(width, height);
mMatBuffer = new MatBuffer(width, height);
decoder.configure(format, surface.getSurface(), null, 0);
break;
}
}
if (decoder == null) {
Log.e("VideoDecoderForOpenCV", "Can't find video info!");
return;
}
valid = true;
}
/**
* Configures encoder and muxer state, and prepares the input Surface. Initializes mEncoder,
* mMuxer, mRecordingSurface, mBufferInfo, mTrackIndex, and mMuxerStarted.
*/
private void configureMediaCodecEncoder() {
mBufferInfo = new MediaCodec.BufferInfo();
MediaFormat format =
MediaFormat.createVideoFormat(MIME_TYPE, mStreamSize.getWidth(), mStreamSize.getHeight());
/**
* Set encoding properties. Failing to specify some of these can cause the MediaCodec
* configure() call to throw an exception.
*/
format.setInteger(
MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
format.setInteger(MediaFormat.KEY_BIT_RATE, mEncBitRate);
format.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE);
format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL);
Log.i(TAG, "configure video encoding format: " + format);
// Create/configure a MediaCodec encoder.
mEncoder = MediaCodec.createEncoderByType(MIME_TYPE);
mEncoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
mRecordingSurface = mEncoder.createInputSurface();
mEncoder.start();
String outputFileName = getOutputMediaFileName();
if (outputFileName == null) {
throw new IllegalStateException("Failed to get video output file");
}
/**
* Create a MediaMuxer. We can't add the video track and start() the muxer until the encoder
* starts and notifies the new media format.
*/
try {
mMuxer = new MediaMuxer(outputFileName, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
} catch (IOException ioe) {
throw new IllegalStateException("MediaMuxer creation failed", ioe);
}
mMuxerStarted = false;
}
// Pass null in |sharedContext| to configure the codec for ByteBuffer output.
private boolean initDecode(VideoCodecType type, int width, int height, EGLContext sharedContext) {
if (mediaCodecThread != null) {
throw new RuntimeException("Forgot to release()?");
}
useSurface = (sharedContext != null);
String mime = null;
String[] supportedCodecPrefixes = null;
if (type == VideoCodecType.VIDEO_CODEC_VP8) {
mime = VP8_MIME_TYPE;
supportedCodecPrefixes = supportedVp8HwCodecPrefixes;
} else if (type == VideoCodecType.VIDEO_CODEC_H264) {
mime = H264_MIME_TYPE;
supportedCodecPrefixes = supportedH264HwCodecPrefixes;
} else {
throw new RuntimeException("Non supported codec " + type);
}
DecoderProperties properties = findDecoder(mime, supportedCodecPrefixes);
if (properties == null) {
throw new RuntimeException("Cannot find HW decoder for " + type);
}
Logging.d(
TAG,
"Java initDecode: "
+ type
+ " : "
+ width
+ " x "
+ height
+ ". Color: 0x"
+ Integer.toHexString(properties.colorFormat)
+ ". Use Surface: "
+ useSurface);
if (sharedContext != null) {
Logging.d(TAG, "Decoder shared EGL Context: " + sharedContext);
}
mediaCodecThread = Thread.currentThread();
try {
Surface decodeSurface = null;
this.width = width;
this.height = height;
stride = width;
sliceHeight = height;
if (useSurface) {
// Create shared EGL context.
eglBase = new EglBase(sharedContext, EglBase.ConfigType.PIXEL_BUFFER);
eglBase.createDummyPbufferSurface();
eglBase.makeCurrent();
// Create output surface
textureID = GlUtil.generateTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES);
Logging.d(TAG, "Video decoder TextureID = " + textureID);
surfaceTexture = new SurfaceTexture(textureID);
surface = new Surface(surfaceTexture);
decodeSurface = surface;
}
MediaFormat format = MediaFormat.createVideoFormat(mime, width, height);
if (!useSurface) {
format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat);
}
Logging.d(TAG, " Format: " + format);
mediaCodec = MediaCodecVideoEncoder.createByCodecName(properties.codecName);
if (mediaCodec == null) {
return false;
}
mediaCodec.configure(format, decodeSurface, null, 0);
mediaCodec.start();
colorFormat = properties.colorFormat;
outputBuffers = mediaCodec.getOutputBuffers();
inputBuffers = mediaCodec.getInputBuffers();
Logging.d(
TAG,
"Input buffers: " + inputBuffers.length + ". Output buffers: " + outputBuffers.length);
return true;
} catch (IllegalStateException e) {
Logging.e(TAG, "initDecode failed", e);
return false;
}
}
@CalledByNative
private static boolean decodeAudioFile(
Context ctx, int nativeMediaCodecBridge, int inputFD, long dataSize) {
if (dataSize < 0 || dataSize > 0x7fffffff) return false;
MediaExtractor extractor = new MediaExtractor();
ParcelFileDescriptor encodedFD;
encodedFD = ParcelFileDescriptor.adoptFd(inputFD);
try {
extractor.setDataSource(encodedFD.getFileDescriptor(), 0, dataSize);
} catch (Exception e) {
e.printStackTrace();
encodedFD.detachFd();
return false;
}
if (extractor.getTrackCount() <= 0) {
encodedFD.detachFd();
return false;
}
MediaFormat format = extractor.getTrackFormat(0);
// Number of channels specified in the file
int inputChannelCount = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
// Number of channels the decoder will provide. (Not
// necessarily the same as inputChannelCount. See
// crbug.com/266006.)
int outputChannelCount = inputChannelCount;
int sampleRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE);
String mime = format.getString(MediaFormat.KEY_MIME);
long durationMicroseconds = 0;
if (format.containsKey(MediaFormat.KEY_DURATION)) {
try {
durationMicroseconds = format.getLong(MediaFormat.KEY_DURATION);
} catch (Exception e) {
Log.d(LOG_TAG, "Cannot get duration");
}
}
if (DEBUG) {
Log.d(
LOG_TAG,
"Tracks: "
+ extractor.getTrackCount()
+ " Rate: "
+ sampleRate
+ " Channels: "
+ inputChannelCount
+ " Mime: "
+ mime
+ " Duration: "
+ durationMicroseconds
+ " microsec");
}
nativeInitializeDestination(
nativeMediaCodecBridge, inputChannelCount, sampleRate, durationMicroseconds);
// Create decoder
MediaCodec codec = MediaCodec.createDecoderByType(mime);
codec.configure(format, null /* surface */, null /* crypto */, 0 /* flags */);
codec.start();
ByteBuffer[] codecInputBuffers = codec.getInputBuffers();
ByteBuffer[] codecOutputBuffers = codec.getOutputBuffers();
// A track must be selected and will be used to read samples.
extractor.selectTrack(0);
boolean sawInputEOS = false;
boolean sawOutputEOS = false;
// Keep processing until the output is done.
while (!sawOutputEOS) {
if (!sawInputEOS) {
// Input side
int inputBufIndex = codec.dequeueInputBuffer(TIMEOUT_MICROSECONDS);
if (inputBufIndex >= 0) {
ByteBuffer dstBuf = codecInputBuffers[inputBufIndex];
int sampleSize = extractor.readSampleData(dstBuf, 0);
long presentationTimeMicroSec = 0;
if (sampleSize < 0) {
sawInputEOS = true;
sampleSize = 0;
} else {
presentationTimeMicroSec = extractor.getSampleTime();
}
codec.queueInputBuffer(
inputBufIndex,
0, /* offset */
sampleSize,
presentationTimeMicroSec,
sawInputEOS ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0);
if (!sawInputEOS) {
extractor.advance();
}
}
}
// Output side
MediaCodec.BufferInfo info = new BufferInfo();
final int outputBufIndex = codec.dequeueOutputBuffer(info, TIMEOUT_MICROSECONDS);
if (outputBufIndex >= 0) {
ByteBuffer buf = codecOutputBuffers[outputBufIndex];
if (info.size > 0) {
nativeOnChunkDecoded(
nativeMediaCodecBridge, buf, info.size, inputChannelCount, outputChannelCount);
}
buf.clear();
codec.releaseOutputBuffer(outputBufIndex, false /* render */);
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
sawOutputEOS = true;
}
} else if (outputBufIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
codecOutputBuffers = codec.getOutputBuffers();
} else if (outputBufIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
MediaFormat newFormat = codec.getOutputFormat();
outputChannelCount = newFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
Log.d(LOG_TAG, "output format changed to " + newFormat);
}
}
encodedFD.detachFd();
codec.stop();
codec.release();
codec = null;
return true;
}