public void read(Buffer buf) {
synchronized (buffers) {
while (buffers.size() == 0 && notDone) {
try {
buffers.wait();
} catch (InterruptedException ex) {
ex.printStackTrace();
}
}
if (buffers.size() > 0) {
Buffer newBuf = (Buffer) buffers.get(0);
int[] newData = (int[]) newBuf.getData();
buf.setData(newData);
buf.setLength(newBuf.getLength());
buf.setOffset(0);
buf.setFormat(vFormat);
buf.setFlags(Buffer.FLAG_KEY_FRAME | Buffer.FLAG_NO_DROP);
buffers.remove(0);
// System.out.println("Removing buffer: size = " + buffers.size());
} else {
buf.setEOM(true);
buf.setOffset(0);
buf.setLength(0);
synchronized (buffers) {
buffers.notify();
}
}
}
}
/**
* Blocks and reads into a <tt>Buffer</tt> from this <tt>PullBufferStream</tt>.
*
* @param buffer the <tt>Buffer</tt> this <tt>PullBufferStream</tt> is to read into
* @throws IOException if an I/O error occurs while this <tt>PullBufferStream</tt> reads into the
* specified <tt>Buffer</tt>
* @see AbstractVideoPullBufferStream#doRead(Buffer)
*/
@Override
protected void doRead(Buffer buffer) throws IOException {
/*
* Determine the Format in which we're expected to output. We cannot
* rely on the Format always being specified in the Buffer because it is
* not its responsibility, the DataSource of this ImageStream knows the
* output Format.
*/
Format format = buffer.getFormat();
if (format == null) {
format = getFormat();
if (format != null) buffer.setFormat(format);
}
if (format instanceof AVFrameFormat) {
Object o = buffer.getData();
AVFrame frame;
if (o instanceof AVFrame) frame = (AVFrame) o;
else {
frame = new AVFrame();
buffer.setData(frame);
}
AVFrameFormat avFrameFormat = (AVFrameFormat) format;
Dimension size = avFrameFormat.getSize();
ByteBuffer data = readScreenNative(size);
if (data != null) {
if (frame.avpicture_fill(data, avFrameFormat) < 0) {
data.free();
throw new IOException("avpicture_fill");
}
} else {
/*
* This can happen when we disconnect a monitor from computer
* before or during grabbing.
*/
throw new IOException("Failed to grab screen.");
}
} else {
byte[] bytes = (byte[]) buffer.getData();
Dimension size = ((VideoFormat) format).getSize();
bytes = readScreen(bytes, size);
buffer.setData(bytes);
buffer.setOffset(0);
buffer.setLength(bytes.length);
}
buffer.setHeader(null);
buffer.setTimeStamp(System.nanoTime());
buffer.setSequenceNumber(seqNo);
buffer.setFlags(Buffer.FLAG_SYSTEM_TIME | Buffer.FLAG_LIVE_DATA);
seqNo++;
}
// @Override
@Override
public int read() throws IOException {
// TODO: how do we detect IOException?
fillBuffer();
if (buffer.getLength() == 0 && buffer.isEOM()) // TODO: will always be
// EOM if length is 0
return -1;
final byte[] data = (byte[]) buffer.getData();
final int result = data[buffer.getOffset()] & 0xff;
buffer.setOffset(buffer.getOffset() + 1);
buffer.setLength(buffer.getLength() - 1);
return result;
}
// @Override
@Override
public int read(byte[] b, int off, int len) throws IOException {
// TODO: how do we detect IOException?
fillBuffer();
if (buffer.getLength() == 0 && buffer.isEOM()) // TODO: will always be
// EOM if length is 0
return -1;
final byte[] data = (byte[]) buffer.getData();
int lengthToCopy = buffer.getLength() < len ? buffer.getLength() : len;
System.arraycopy(data, buffer.getOffset(), b, off, lengthToCopy);
buffer.setOffset(buffer.getOffset() + lengthToCopy);
buffer.setLength(buffer.getLength() - lengthToCopy);
return lengthToCopy;
}
/**
* Reads media data from this <tt>PullBufferStream</tt> into a specific <tt>Buffer</tt> with
* blocking.
*
* @param buffer the <tt>Buffer</tt> in which media data is to be read from this
* <tt>PullBufferStream</tt>
* @throws IOException if anything goes wrong while reading media data from this
* <tt>PullBufferStream</tt> into the specified <tt>buffer</tt>
* @see javax.media.protocol.PullBufferStream#read(javax.media.Buffer)
*/
public void read(Buffer buffer) throws IOException {
if (setThreadPriority) {
setThreadPriority = false;
setThreadPriority();
}
Object data = buffer.getData();
int length = this.length;
if (data instanceof byte[]) {
if (((byte[]) data).length < length) data = null;
} else data = null;
if (data == null) {
data = new byte[length];
buffer.setData(data);
}
int toRead = length;
byte[] bytes = (byte[]) data;
int offset = 0;
buffer.setLength(0);
while (toRead > 0) {
int read;
synchronized (this) {
if (audioRecord.getRecordingState() == AudioRecord.RECORDSTATE_RECORDING)
read = audioRecord.read(bytes, offset, toRead);
else break;
}
if (read < 0) {
throw new IOException(
AudioRecord.class.getName() + "#read(byte[], int, int) returned " + read);
} else {
buffer.setLength(buffer.getLength() + read);
offset += read;
toRead -= read;
}
}
buffer.setOffset(0);
// Apply software gain.
if (gainControl != null) {
BasicVolumeControl.applyGain(gainControl, bytes, buffer.getOffset(), buffer.getLength());
}
}
/** decode the buffer * */
public int process(Buffer inputBuffer, Buffer outputBuffer) {
if (!checkInputBuffer(inputBuffer)) {
return BUFFER_PROCESSED_FAILED;
}
if (isEOM(inputBuffer)) {
propagateEOM(outputBuffer);
return BUFFER_PROCESSED_OK;
}
Object outData = outputBuffer.getData();
outputBuffer.setData(inputBuffer.getData());
inputBuffer.setData(outData);
outputBuffer.setLength(inputBuffer.getLength());
outputBuffer.setFormat(outputFormat);
outputBuffer.setOffset(inputBuffer.getOffset());
return BUFFER_PROCESSED_OK;
}
@Override
public int process(Buffer input, Buffer output) {
if (!checkInputBuffer(input)) {
return BUFFER_PROCESSED_FAILED;
}
if (isEOM(input)) {
propagateEOM(output); // TODO: what about data? can there be any?
return BUFFER_PROCESSED_OK;
}
try {
// TODO: this is very inefficient - it allocates a new byte array
// (or more) every time
final ByteArrayInputStream is =
new ByteArrayInputStream((byte[]) input.getData(), input.getOffset(), input.getLength());
final BufferedImage image = ImageIO.read(is);
is.close();
final Buffer b =
ImageToBuffer.createBuffer(image, ((VideoFormat) outputFormat).getFrameRate());
output.setData(b.getData());
output.setOffset(b.getOffset());
output.setLength(b.getLength());
output.setFormat(b.getFormat()); // TODO: this is a bit hacky, this
// format will be more specific
// than the actual set output
// format, because now we know what
// ImageIO gave us for a
// BufferedImage as far as pixel
// masks, etc.
return BUFFER_PROCESSED_OK;
} catch (IOException e) {
output.setDiscard(true);
output.setLength(0);
return BUFFER_PROCESSED_FAILED;
}
}
@Override
public void readFrame(Buffer buffer) {
// example data:
// --ssBoundary8345
// Content-Type: image/jpeg
// Content-Length: 114587
try {
String line;
// eat leading blank lines
while (true) {
line = readLine(MAX_LINE_LENGTH);
if (line == null) {
buffer.setEOM(true);
buffer.setLength(0);
return;
}
if (!line.trim().equals("")) break; // end of header
}
if (boundary == null) {
boundary = line.trim(); // TODO: we should be able to get
// this from the content type, but
// the content type has this
// stripped out. So we'll just take
// the first nonblank line to be the
// boundary.
// System.out.println("boundary: " + boundary);
} else {
if (!line.trim().equals(boundary)) {
// throw new IOException("Expected boundary: " +
// toPrintable(line));
// TODO: why do we seem to get these when playing back
// mmr files recorded using FmjTranscode?
logger.warning("Expected boundary (frame " + framesRead + "): " + toPrintable(line));
// handle streams that are truncated in the middle of a
// frame:
final int eatResult = eatUntil(boundary); // TODO: no
// need to
// store the
// data
logger.info(
"Ignored bytes (eom after="
+ (eatResult < 0)
+ "): "
+ (eatResult < 0 ? (-1 * eatResult - 1) : eatResult));
if (eatResult < 0) {
buffer.setEOM(true);
buffer.setLength(0);
return;
}
// now read boundary
line = readLine(MAX_LINE_LENGTH);
if (!line.trim().equals(boundary)) {
throw new RuntimeException("No boundary found after eatUntil(boundary)"); // should
// never
// happen
}
}
}
final Properties properties = new Properties();
while (true) {
line = readLine(MAX_LINE_LENGTH);
if (line == null) {
buffer.setEOM(true);
buffer.setLength(0);
return;
}
if (line.trim().equals("")) break; // end of header
if (!parseProperty(line, properties))
throw new IOException("Expected property: " + toPrintable(line));
}
final String contentType = properties.getProperty("Content-Type".toUpperCase());
if (contentType == null) {
logger.warning("Header properties: " + properties);
throw new IOException("Expected Content-Type in header");
}
// check supported content types:
if (!isSupportedFrameContentType(contentType)) {
throw new IOException("Unsupported Content-Type: " + contentType);
}
if (frameContentType == null) {
frameContentType = contentType;
} else {
if (!contentType.equals(frameContentType))
throw new IOException(
"Content type changed during stream from "
+ frameContentType
+ " to "
+ contentType);
}
// TODO: check that size doesn't change throughout
final byte[] data;
final String contentLenStr = properties.getProperty("Content-Length".toUpperCase());
if (contentLenStr != null) { // if we know the content length, use it
final int contentLen;
try {
contentLen = Integer.parseInt(contentLenStr);
} catch (NumberFormatException e) {
throw new IOException("Invalid content length: " + contentLenStr);
}
// now, read the content-length bytes
data = readFully(contentLen); // TODO: don't realloc each
// time
} else {
// if we don't know the content length, just read until we
// find the boundary.
// Some IP cameras don't specify it, like
// http://webcam-1.duesseldorf.it-on.net/cgi-bin/nph-update.cgi
data = readUntil(boundary);
}
// ext
final String timestampStr = properties.getProperty(TIMESTAMP_KEY.toUpperCase());
if (timestampStr != null) {
try {
final long timestamp = Long.parseLong(timestampStr);
buffer.setTimeStamp(timestamp);
} catch (NumberFormatException e) {
logger.log(Level.WARNING, "" + e, e);
}
}
if (data == null) {
buffer.setEOM(true);
buffer.setLength(0);
return;
}
buffer.setData(data);
buffer.setOffset(0);
buffer.setLength(data.length);
++framesRead;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/** {@inheritDoc} */
@Override
protected int doProcess(Buffer inBuffer, Buffer outBuffer) {
byte[] inData = (byte[]) inBuffer.getData();
int inOffset = inBuffer.getOffset();
if (!VP8PayloadDescriptor.isValid(inData, inOffset)) {
logger.warn("Invalid RTP/VP8 packet discarded.");
outBuffer.setDiscard(true);
return BUFFER_PROCESSED_FAILED; // XXX: FAILED or OK?
}
long inSeq = inBuffer.getSequenceNumber();
long inRtpTimestamp = inBuffer.getRtpTimeStamp();
int inPictureId = VP8PayloadDescriptor.getPictureId(inData, inOffset);
boolean inMarker = (inBuffer.getFlags() & Buffer.FLAG_RTP_MARKER) != 0;
boolean inIsStartOfFrame = VP8PayloadDescriptor.isStartOfFrame(inData, inOffset);
int inLength = inBuffer.getLength();
int inPdSize = VP8PayloadDescriptor.getSize(inData, inOffset);
int inPayloadLength = inLength - inPdSize;
if (empty && lastSentSeq != -1 && seqNumComparator.compare(inSeq, lastSentSeq) != 1) {
if (logger.isInfoEnabled()) logger.info("Discarding old packet (while empty) " + inSeq);
outBuffer.setDiscard(true);
return BUFFER_PROCESSED_OK;
}
if (!empty) {
// if the incoming packet has a different PictureID or timestamp
// than those of the current frame, then it belongs to a different
// frame.
if ((inPictureId != -1 && pictureId != -1 && inPictureId != pictureId)
| (timestamp != -1 && inRtpTimestamp != -1 && inRtpTimestamp != timestamp)) {
if (seqNumComparator.compare(inSeq, firstSeq) != 1) // inSeq <= firstSeq
{
// the packet belongs to a previous frame. discard it
if (logger.isInfoEnabled()) logger.info("Discarding old packet " + inSeq);
outBuffer.setDiscard(true);
return BUFFER_PROCESSED_OK;
} else // inSeq > firstSeq (and also presumably isSeq > lastSeq)
{
// the packet belongs to a subsequent frame (to the one
// currently being held). Drop the current frame.
if (logger.isInfoEnabled())
logger.info(
"Discarding saved packets on arrival of"
+ " a packet for a subsequent frame: "
+ inSeq);
// TODO: this would be the place to complain about the
// not-well-received PictureID by sending a RTCP SLI or NACK.
reinit();
}
}
}
// a whole frame in a single packet. avoid the extra copy to
// this.data and output it immediately.
if (empty && inMarker && inIsStartOfFrame) {
byte[] outData = validateByteArraySize(outBuffer, inPayloadLength, false);
System.arraycopy(inData, inOffset + inPdSize, outData, 0, inPayloadLength);
outBuffer.setOffset(0);
outBuffer.setLength(inPayloadLength);
outBuffer.setRtpTimeStamp(inBuffer.getRtpTimeStamp());
if (TRACE) logger.trace("Out PictureID=" + inPictureId);
lastSentSeq = inSeq;
return BUFFER_PROCESSED_OK;
}
// add to this.data
Container container = free.poll();
if (container == null) container = new Container();
if (container.buf == null || container.buf.length < inPayloadLength)
container.buf = new byte[inPayloadLength];
if (data.get(inSeq) != null) {
if (logger.isInfoEnabled())
logger.info("(Probable) duplicate packet detected, discarding " + inSeq);
outBuffer.setDiscard(true);
return BUFFER_PROCESSED_OK;
}
System.arraycopy(inData, inOffset + inPdSize, container.buf, 0, inPayloadLength);
container.len = inPayloadLength;
data.put(inSeq, container);
// update fields
frameLength += inPayloadLength;
if (firstSeq == -1 || (seqNumComparator.compare(firstSeq, inSeq) == 1)) firstSeq = inSeq;
if (lastSeq == -1 || (seqNumComparator.compare(inSeq, lastSeq) == 1)) lastSeq = inSeq;
if (empty) {
// the first received packet for the current frame was just added
empty = false;
timestamp = inRtpTimestamp;
pictureId = inPictureId;
}
if (inMarker) haveEnd = true;
if (inIsStartOfFrame) haveStart = true;
// check if we have a full frame
if (frameComplete()) {
byte[] outData = validateByteArraySize(outBuffer, frameLength, false);
int ptr = 0;
Container b;
for (Map.Entry<Long, Container> entry : data.entrySet()) {
b = entry.getValue();
System.arraycopy(b.buf, 0, outData, ptr, b.len);
ptr += b.len;
}
outBuffer.setOffset(0);
outBuffer.setLength(frameLength);
outBuffer.setRtpTimeStamp(inBuffer.getRtpTimeStamp());
if (TRACE) logger.trace("Out PictureID=" + inPictureId);
lastSentSeq = lastSeq;
// prepare for the next frame
reinit();
return BUFFER_PROCESSED_OK;
} else {
// frame not complete yet
outBuffer.setDiscard(true);
return OUTPUT_BUFFER_NOT_FILLED;
}
}
/** Write the buffer to the SourceDataLine. */
public int process(Buffer buffer) {
// if we need to convert the format, do so using the codec.
if (codec != null) {
final int codecResult = codec.process(buffer, codecBuffer);
if (codecResult == BUFFER_PROCESSED_FAILED) return BUFFER_PROCESSED_FAILED;
if (codecResult == OUTPUT_BUFFER_NOT_FILLED) return BUFFER_PROCESSED_OK;
buffer = codecBuffer;
}
int length = buffer.getLength();
int offset = buffer.getOffset();
final Format format = buffer.getFormat();
final Class type = format.getDataType();
if (type != Format.byteArray) {
return BUFFER_PROCESSED_FAILED;
}
final byte[] data = (byte[]) buffer.getData();
final boolean bufferNotConsumed;
final int newBufferLength; // only applicable if bufferNotConsumed
final int newBufferOffset; // only applicable if bufferNotConsumed
if (NON_BLOCKING) {
// TODO: handle sourceLine.available(). This code currently causes choppy audio.
if (length > sourceLine.available()) {
// we should only write sourceLine.available() bytes, then return INPUT_BUFFER_NOT_CONSUMED.
length = sourceLine.available(); // don't try to write more than available
bufferNotConsumed = true;
newBufferLength = buffer.getLength() - length;
newBufferOffset = buffer.getOffset() + length;
} else {
bufferNotConsumed = false;
newBufferLength = length;
newBufferOffset = offset;
}
} else {
bufferNotConsumed = false;
newBufferLength = 0;
newBufferOffset = 0;
}
if (length == 0) {
logger.finer("Buffer has zero length, flags = " + buffer.getFlags());
}
// make sure all the bytes are written.
while (length > 0) {
// logger.fine("Available: " + sourceLine.available());
// logger.fine("length: " + length);
// logger.fine("sourceLine.getBufferSize(): " + sourceLine.getBufferSize());
final int n =
sourceLine.write(
data, offset, length); // TODO: this can block for a very long time if it doesn't
if (n >= length) break;
else if (n == 0) {
// TODO: we could choose to handle a write failure this way,
// assuming that it is considered legal to call stop while process is being called.
// however, that seems like a bad idea in general.
// if (!sourceLine.isRunning())
// {
// buffer.setLength(offset);
// buffer.setOffset(length);
// return INPUT_BUFFER_NOT_CONSUMED; // our write was interrupted.
// }
logger.warning(
"sourceLine.write returned 0, offset="
+ offset
+ "; length="
+ length
+ "; available="
+ sourceLine.available()
+ "; frame size in bytes"
+ sourceLine.getFormat().getFrameSize()
+ "; sourceLine.isActive() = "
+ sourceLine.isActive()
+ "; "
+ sourceLine.isOpen()
+ "; sourceLine.isRunning()="
+ sourceLine.isRunning());
return BUFFER_PROCESSED_FAILED; // sourceLine.write docs indicate that this will only happen
// if there is an error.
} else {
offset += n;
length -= n;
}
}
if (bufferNotConsumed) {
// return INPUT_BUFFER_NOT_CONSUMED if not all bytes were written
buffer.setLength(newBufferLength);
buffer.setOffset(newBufferOffset);
return INPUT_BUFFER_NOT_CONSUMED;
}
if (buffer.isEOM()) {
// TODO: the proper way to do this is to implement Drainable, and let the processor call our
// drain method.
sourceLine
.drain(); // we need to ensure that the media finishes playing, otherwise the EOM event
// will
// be posted before the media finishes playing.
}
return BUFFER_PROCESSED_OK;
}