private int doRead(ReadableByteChannel channel, ByteBuffer byteBuffer) throws IOException {
int readBytes = 0;
int lastReadBytes = -1;
try {
do {
lastReadBytes = channel.read(byteBuffer);
if (lastReadBytes > 0) {
readBytes += lastReadBytes;
} else if (lastReadBytes == -1 && readBytes == 0) {
readBytes = -1;
}
} while (lastReadBytes > 0 && byteBuffer.hasRemaining());
} catch (IOException ex) {
lastReadBytes = -1;
throw ex;
} finally {
if (lastReadBytes == -1 || readBytes == -1) {
SelectionKeyHandler skh = selectorHandler.getSelectionKeyHandler();
if (skh instanceof BaseSelectionKeyHandler) {
((BaseSelectionKeyHandler) skh)
.notifyRemotlyClose(
((SelectableChannel) channel).keyFor(selectorHandler.getSelector()));
}
}
}
return readBytes;
}
@org.junit.Test
public void testMapper() throws Exception {
final ArcFileReader reader = new ArcFileReader();
Thread thread =
new Thread(
new Runnable() {
public void run() {
try {
while (reader.hasMoreItems()) {
ArcFileItem item = new ArcFileItem();
reader.getNextItem(item);
map(new Text(item.getUri()), item, null, null);
}
LOG.info("NO MORE ITEMS... BYE");
} catch (IOException e) {
LOG.error(StringUtils.stringifyException(e));
}
}
});
// run the thread ...
thread.start();
File file = new File("/Users/rana/Downloads/1213886083018_0.arc.gz");
ReadableByteChannel channel = Channels.newChannel(new FileInputStream(file));
try {
int totalBytesRead = 0;
for (; ; ) {
ByteBuffer buffer = ByteBuffer.allocate(ArcFileReader.DEFAULT_BLOCK_SIZE);
int bytesRead = channel.read(buffer);
LOG.info("Read " + bytesRead + " From File");
if (bytesRead == -1) {
reader.finished();
break;
} else {
buffer.flip();
totalBytesRead += buffer.remaining();
reader.available(buffer);
}
}
} finally {
channel.close();
}
// now wait for thread to die ...
LOG.info("Done Reading File.... Waiting for ArcFileThread to DIE");
thread.join();
LOG.info("Done Reading File.... ArcFileThread to DIED");
}
static int safeRead(ReadableByteChannel channel, ByteBuffer dst) throws IOException {
int read = -1;
try {
// Read data from the Channel
read = channel.read(dst);
} catch (ClosedChannelException e) {
} catch (IOException e) {
switch ("" + e.getMessage()) {
case "null":
case "Connection reset by peer":
case "Broken pipe":
break;
default:
cntIoError++;
System.out.println("error count IoError: " + cntIoError);
}
channel.close();
e.printStackTrace();
} catch (CancelledKeyException e) {
cntCancelledKeyException++;
System.out.println("error count CancelledKey: " + cntCancelledKeyException);
channel.close();
}
return read;
}
/**
* This is a wrapper around {@link ReadableByteChannel#read(ByteBuffer)}. If the amount of data is
* large, it writes to channel in smaller chunks. This is to avoid jdk from creating many direct
* buffers as the size of ByteBuffer increases. There should not be any performance degredation.
*
* @see ReadableByteChannel#read(ByteBuffer)
*/
private static int channelRead(ReadableByteChannel channel, ByteBuffer buffer)
throws IOException {
return (buffer.remaining() <= NIO_BUFFER_LIMIT)
? channel.read(buffer)
: channelIO(channel, null, buffer);
}
/**
* Helper for {@link #channelRead(ReadableByteChannel, ByteBuffer)} and {@link
* #channelWrite(WritableByteChannel, ByteBuffer)}. Only one of readCh or writeCh should be
* non-null.
*
* @see #channelRead(ReadableByteChannel, ByteBuffer)
* @see #channelWrite(WritableByteChannel, ByteBuffer)
*/
private static int channelIO(
ReadableByteChannel readCh, WritableByteChannel writeCh, ByteBuffer buf) throws IOException {
int originalLimit = buf.limit();
int initialRemaining = buf.remaining();
int ret = 0;
while (buf.remaining() > 0) {
try {
int ioSize = Math.min(buf.remaining(), NIO_BUFFER_LIMIT);
buf.limit(buf.position() + ioSize);
ret = (readCh == null) ? writeCh.write(buf) : readCh.read(buf);
if (ret < ioSize) {
break;
}
} finally {
buf.limit(originalLimit);
}
}
int nBytes = initialRemaining - buf.remaining();
return (nBytes > 0) ? nBytes : ret;
}
/**
* Platform-independent channel-to-channel transfer method. Uses regular {@code read} and {@code
* write} operations to move bytes from the {@code source} channel to the {@code sink} channel.
* After this call, the {@code throughBuffer} should be checked for remaining bytes; if there are
* any, they should be written to the {@code sink} channel before proceeding. This method may be
* used with NIO channels, XNIO channels, or a combination of the two.
*
* <p>If either or both of the given channels are blocking channels, then this method may block.
*
* @param source the source channel to read bytes from
* @param count the number of bytes to transfer (must be >= {@code 0L})
* @param throughBuffer the buffer to transfer through (must not be {@code null})
* @param sink the sink channel to write bytes to
* @return the number of bytes actually transferred (possibly 0)
* @throws IOException if an I/O error occurs during the transfer of bytes
*/
public static long transfer(
final ReadableByteChannel source,
final long count,
final ByteBuffer throughBuffer,
final WritableByteChannel sink)
throws IOException {
long res;
long total = 0L;
throughBuffer.limit(0);
while (total < count) {
throughBuffer.compact();
try {
if (count - total < (long) throughBuffer.remaining()) {
throughBuffer.limit((int) (count - total));
}
res = source.read(throughBuffer);
if (res <= 0) {
return total == 0L ? res : total;
}
} finally {
throughBuffer.flip();
}
res = sink.write(throughBuffer);
if (res == 0) {
return total;
}
total += res;
}
return total;
}
public void write(
String path, ReadableByteChannel data, ByteBuffer temporaryBuffer, boolean hasData)
throws IOException {
try {
File file = new File(basePath, path);
RandomAccessFile randomAccessFile = null;
try {
file.getParentFile().mkdirs();
randomAccessFile = new RandomAccessFile(file, "rw");
if (hasData) {
FileChannel channel = randomAccessFile.getChannel();
while (data.read(temporaryBuffer) >= 0) {
temporaryBuffer.flip();
channel.write(temporaryBuffer);
temporaryBuffer.clear();
}
}
} finally {
if (randomAccessFile != null) {
randomAccessFile.close();
}
}
} catch (Throwable t) {
t.printStackTrace();
throw new IOException(t);
}
}
/**
* Copy the readable byte channel to the writable byte channel
*
* @param inChannel the readable byte channel
* @param outChannel the writable byte channel
* @throws IOException if an IOException occurs.
*/
public static void copy(ReadableByteChannel inChannel, WritableByteChannel outChannel)
throws IOException {
if (inChannel instanceof FileChannel) {
((FileChannel) inChannel).transferTo(0, ((FileChannel) inChannel).size(), outChannel);
} else {
final ByteBuffer buffer = ByteBuffer.allocateDirect(16 * 1024);
try {
while (inChannel.read(buffer) != -1) {
// prepare the buffer to be drained
buffer.flip();
// write to the channel, may block
outChannel.write(buffer);
// If partial transfer, shift remainder down
// If buffer is empty, same as doing clear()
buffer.compact();
}
// EOF will leave buffer in fill state
buffer.flip();
// make sure the buffer is fully drained.
while (buffer.hasRemaining()) {
outChannel.write(buffer);
}
} finally {
IOUtils.close(inChannel);
IOUtils.close(outChannel);
}
}
}
/**
* Helper Method to Copy from one Byte Channel to another.
*
* @param from
* @param to
* @throws IOException
*/
protected static void channelCopy(ReadableByteChannel from, WritableByteChannel to)
throws IOException {
ByteBuffer buffer = ByteBuffer.allocateDirect(16 * 1024);
try {
// Read
while (from.read(buffer) != -1) {
buffer.flip();
to.write(buffer);
buffer.compact();
}
// Flip the Buffer
buffer.flip();
// Write
while (buffer.hasRemaining()) {
to.write(buffer);
} // End of While Loop
} finally {
// Handle In Channel Closure
if (from != null) {
try {
from.close();
} catch (Exception ex) {
// No handling required
}
}
// Handle Out Channel Closure
if (to != null) {
try {
to.close();
} catch (Exception ex) {
// No handling required
}
}
} // End of Finally
}
public static long stream(InputStream input, OutputStream output) throws IOException {
ReadableByteChannel inputChannel = null;
WritableByteChannel outputChannel = null;
try {
inputChannel = Channels.newChannel(input);
outputChannel = Channels.newChannel(output);
ByteBuffer buffer = ByteBuffer.allocate(10240);
long size = 0;
while (inputChannel.read(buffer) != -1) {
buffer.flip();
size += outputChannel.write(buffer);
buffer.clear();
}
return size;
} finally {
if (outputChannel != null)
try {
outputChannel.close();
} catch (IOException ignore) {
/**/
}
if (inputChannel != null)
try {
inputChannel.close();
} catch (IOException ignore) {
/**/
}
}
}
/**
* transfer Stream
*
* @param ins
* @param targetChannel
*/
private static void transferStream(InputStream ins, FileChannel targetChannel) {
ByteBuffer byteBuffer = ByteBuffer.allocate(1024 * 10);
ReadableByteChannel rbcInst = Channels.newChannel(ins);
try {
while (-1 != (rbcInst.read(byteBuffer))) {
byteBuffer.flip();
targetChannel.write(byteBuffer);
byteBuffer.clear();
}
} catch (IOException ioe) {
ioe.printStackTrace();
} finally {
if (null != rbcInst) {
try {
rbcInst.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (null != targetChannel) {
try {
targetChannel.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
public String copy(String basePath, ImageRef source, String targetDirName) {
try {
ByteBuffer buffer = ByteBuffer.allocate(16 * 1024);
Resource res = source.eResource();
URI uri = source.eResource().getURI();
URI inPath = URI.createURI(source.getPath()).resolve(uri);
URI outPath =
URI.createURI(source.getPath())
.resolve(URI.createFileURI(new File(targetDirName).getAbsolutePath()));
ReadableByteChannel inChannel =
Channels.newChannel(res.getResourceSet().getURIConverter().createInputStream(inPath));
WritableByteChannel outChannel =
Channels.newChannel(res.getResourceSet().getURIConverter().createOutputStream(outPath));
while (inChannel.read(buffer) != -1) {
buffer.flip();
outChannel.write(buffer);
buffer.compact();
}
buffer.flip();
while (buffer.hasRemaining()) {
outChannel.write(buffer);
}
outChannel.close();
return outPath.toFileString();
} catch (IOException e) {
// TODO Auto-generated catch block
logger.error(e);
return null;
}
}
public static int readToBuffer(ReadableByteChannel channel, ByteBuffer buffer)
throws NetIOException {
try {
return channel.read(buffer);
} catch (IOException e) {
return -1;
}
}
@Test(expected = IOException.class)
public void testTruncatedData() throws IOException {
// with BZ2File(self.filename) as f:
// self.assertRaises(EOFError, f.read)
System.out.println("Attempt to read the whole thing in, should throw ...");
ByteBuffer buffer = ByteBuffer.allocate(8192);
bz2Channel.read(buffer);
}
/**
* Reads the given website with the URL {@code url} or downloads its
* contents and returns them as a {@link String}.
*
* @param url
* the URL
* @return the lines
*/
public static String readWebsite(String url) throws IOException
{
URL url1 = new URL(url);
ReadableByteChannel rbc = Channels.newChannel(url1.openStream());
ByteBuffer bytebuf = ByteBuffer.allocate(1024);
rbc.read(bytebuf);
return new String(bytebuf.array());
}
/**
* Reads the data from the source and writes it to the destination
*
* @param src The source to read data from
* @param trg The target to write data to
*/
public static void inputToOutput(ReadableByteChannel src, WritableByteChannel trg)
throws IOException {
ByteBuffer bb = ByteBuffer.allocateDirect(4096);
while (src.read(bb) != -1) {
bb.flip();
while (bb.hasRemaining()) trg.write(bb);
bb.clear();
}
}
/** Gets String contents from channel and closes it. */
public static String getStringContents(ReadableByteChannel channel) throws IOException {
// TODO Checks if a supplier would be nice
try {
ByteBuffer buffer = ByteBuffer.allocate(1024 * 8);
StringBuilder sb = new StringBuilder();
int bytesRead = channel.read(buffer);
while (bytesRead != -1) {
buffer.flip();
CharBuffer charBuffer = StandardCharsets.UTF_8.decode(buffer);
sb.append(charBuffer.toString());
buffer.clear();
bytesRead = channel.read(buffer);
}
return sb.toString();
} finally {
channel.close();
}
}
/**
* Refills the byte buffer.
*
* @param sourceChannel The byte channel to read from.
* @return The number of bytes read and added or -1 if end of the source channel reached.
* @throws IOException
*/
public int fill(ReadableByteChannel sourceChannel) throws IOException {
int result = 0;
if (sourceChannel.isOpen()) {
result = sourceChannel.read(getBytes());
}
return result;
}
// for filling a ReadableByteChannel
public static int fill(ByteBuffer buffer, ReadableByteChannel channel) throws IOException {
int r = buffer.remaining();
// channel reads return -1 when EOF or other error
// because they a non-blocking reads, 0 is a valid return value!!
while (buffer.remaining() > 0 && r != -1) {
r = channel.read(buffer);
}
buffer.limit(buffer.position());
return r;
}
/**
* A short cut for reading the header from the given channel.
*
* @param channel The channel to read from.
* @param warningListener
* @throws IOException If problems arise.
* @return A ShapefileHeader object.
*/
public static ShapefileHeader readHeader(
ReadableByteChannel channel, WarningListener warningListener) throws IOException {
ByteBuffer buffer = ByteBuffer.allocateDirect(100);
if (channel.read(buffer) != 100) {
throw new EOFException("Premature end of header");
}
buffer.flip();
ShapefileHeader header = new ShapefileHeader();
header.read(buffer, warningListener);
return header;
}
@Test
public void testPartialReadTruncatedData() throws IOException {
// with BZ2File(self.filename) as f:
// self.assertEqual(f.read(len(self.TEXT)), self.TEXT)
// self.assertRaises(EOFError, f.read, 1)
final int length = TEXT.length();
ByteBuffer buffer = ByteBuffer.allocate(length);
bz2Channel.read(buffer);
assertArrayEquals(copyOfRange(TEXT.getBytes(), 0, length), buffer.array());
// subsequent read should throw
buffer = ByteBuffer.allocate(1);
try {
bz2Channel.read(buffer);
Assert.fail("The read should have thrown.");
} catch (IOException e) {
// pass
}
}
private static void channelCopy2(ReadableByteChannel src, WritableByteChannel dest)
throws IOException {
ByteBuffer buffer = ByteBuffer.allocateDirect(16 * 1024);
while (src.read(buffer) != -1) {
buffer.flip();
while (buffer.hasRemaining()) {
dest.write(buffer);
}
buffer.clear();
}
}
/** Sending and receiving data by the socket */
@Override
void handle_poll_activity(int timeout) {
if (!socket_.isConnected()) {
Log.d(TAG, "Socket is not connected");
break_contact(ContactEvent.reason_t.BROKEN);
}
// if we have something to send , send it first
if (sendbuf_.position() > 0) send_data();
// poll to receive and process data
try {
num_to_read_ = read_stream_.available();
// check that there's something to read
if (num_to_read_ > 0) {
Log.d(TAG, "before reading position is " + recvbuf_.position());
java.nio.ByteBuffer temp_java_nio_buf = java.nio.ByteBuffer.allocate(recvbuf_.remaining());
read_channel_.read(temp_java_nio_buf);
BufferHelper.copy_data(
recvbuf_, recvbuf_.position(), temp_java_nio_buf, 0, temp_java_nio_buf.position());
recvbuf_.position(recvbuf_.position() + temp_java_nio_buf.position());
if (DTNService.is_test_data_logging())
TestDataLogger.getInstance()
.set_downloaded_size(
TestDataLogger.getInstance().downloaded_size() + temp_java_nio_buf.position());
Log.d(TAG, "buffer position now is " + recvbuf_.position());
process_data();
if (recvbuf_.remaining() == 0) {
Log.e(TAG, "after process_data left no space in recvbuf!!");
}
}
} catch (IOException e) {
Log.e(TAG, "IOException, in reading data from the read_stream_:" + e.getMessage());
}
// send keep alive message if we should send it
if (contact_up_ && !contact_broken_) {
check_keepalive();
}
if (!contact_broken_) check_timeout();
}
private static void dumpTSPackets(ReadableByteChannel _in) throws IOException {
ByteBuffer buf = ByteBuffer.allocate(188 * 1024);
while (_in.read(buf) != -1) {
buf.flip();
buf.limit((buf.limit() / 188) * 188);
int pmtPid = -1;
for (int pkt = 0; buf.hasRemaining(); ++pkt) {
ByteBuffer tsBuf = NIOUtils.read(buf, 188);
Assert.assertEquals(0x47, tsBuf.get() & 0xff);
int guidFlags = ((tsBuf.get() & 0xff) << 8) | (tsBuf.get() & 0xff);
int guid = (int) guidFlags & 0x1fff;
int payloadStart = (guidFlags >> 14) & 0x1;
int b0 = tsBuf.get() & 0xff;
int counter = b0 & 0xf;
if ((b0 & 0x20) != 0) {
NIOUtils.skip(tsBuf, (tsBuf.get() & 0xff));
}
System.out.print(
"#"
+ pkt
+ "[guid: "
+ guid
+ ", cnt: "
+ counter
+ ", start: "
+ (payloadStart == 1 ? "y" : "-"));
if (guid == 0 || guid == pmtPid) {
System.out.print(", PSI]: ");
if (payloadStart == 1) {
NIOUtils.skip(tsBuf, (tsBuf.get() & 0xff));
}
if (guid == 0) {
PATSection pat = PATSection.parsePAT(tsBuf);
IntIntMap programs = pat.getPrograms();
pmtPid = programs.values()[0];
printPat(pat);
} else if (guid == pmtPid) {
PMTSection pmt = PMTSection.parsePMT(tsBuf);
printPmt(pmt);
}
} else {
System.out.print("]: " + tsBuf.remaining());
}
System.out.println();
}
buf.clear();
}
}
/**
* Perform a low-level read of the remaining number of bytes into the specified byte buffer. The
* incoming bytes will be used to fill the remaining space in the target byte buffer. This is
* equivalent to the read(2) POSIX function, and like that function it ignores read and write
* buffers defined elsewhere.
*
* @param buffer the java.nio.ByteBuffer in which to put the incoming bytes
* @return the number of bytes actually read
* @throws java.io.IOException if there is an exception during IO
* @throws org.jruby.util.io.BadDescriptorException if the associated channel is already closed
* @see java.nio.ByteBuffer
*/
public int read(ByteBuffer buffer) throws IOException, BadDescriptorException {
checkOpen();
// TODO: It would be nice to throw a better error for this
if (!isReadable()) {
throw new BadDescriptorException();
}
ReadableByteChannel readChannel = (ReadableByteChannel) channel;
int bytesRead = 0;
bytesRead = readChannel.read(buffer);
return bytesRead;
}
/**
* Copies all bytes from the readable channel to the writable channel. Does not close or flush
* either channel.
*
* @param from the readable channel to read from
* @param to the writable channel to write to
* @return the number of bytes copied
* @throws IOException if an I/O error occurs
*/
public static long copy(ReadableByteChannel from, WritableByteChannel to) throws IOException {
checkNotNull(from);
checkNotNull(to);
ByteBuffer buf = ByteBuffer.allocate(BUF_SIZE);
long total = 0;
while (from.read(buf) != -1) {
buf.flip();
while (buf.hasRemaining()) {
total += to.write(buf);
}
buf.clear();
}
return total;
}
public static void main(String[] args) throws IOException {
FileInputStream fin = new FileInputStream(args[0]);
GZIPInputStream gzin = new GZIPInputStream(fin);
ReadableByteChannel in = Channels.newChannel(gzin);
WritableByteChannel out = Channels.newChannel(System.out);
ByteBuffer buffer = ByteBuffer.allocate(65536);
while (in.read(buffer) != -1) {
buffer.flip();
out.write(buffer);
buffer.clear();
}
}
// static helper methods
public static int channelRead(ReadableByteChannel channel, ByteBuffer buf) throws IOException {
if (buf.remaining() <= NIO_BUFFER_LIMIT) {
return channel.read(buf);
}
int originalLimit = buf.limit();
int initialRemaining = buf.remaining();
int ret = 0;
while (buf.remaining() > 0) {
try {
int ioSize = Math.min(buf.remaining(), NIO_BUFFER_LIMIT);
buf.limit(buf.position() + ioSize);
ret = channel.read(buf);
if (ret < ioSize) {
break;
}
} finally {
buf.limit(originalLimit);
}
}
int nBytes = initialRemaining - buf.remaining();
return (nBytes > 0) ? nBytes : ret;
}
/**
* Load the byte code for a class from a file based on the path and class name. Created: Mar 30,
* 2004
*
* @param fileLocator
* @return - the byte code for the class.
*/
private byte[] loadFileData(ProgramResourceLocator.File fileLocator) {
InputStream fileContents = null;
try {
fileContents =
resourceRepository.getContents(
fileLocator); // throws IOException if the file doesn't exist.
ReadableByteChannel channel = Channels.newChannel(fileContents);
// Note: Another way to do this is to determine the channel size, and allocate an array of the
// correct size.
// However, this involves additional file system access.
// Init the output buffer for holding the input stream
ByteArrayOutputStream baos = threadLocalByteArrayOutputStream.get();
baos.reset();
byte[] classData = threadLocalByteArray.get();
ByteBuffer buffer = ByteBuffer.wrap(classData);
// Read the input stream to a buffer and write the buffer to the output buffer
while (true) {
int bytesInBuffer = channel.read(buffer);
if (bytesInBuffer < 0) {
break;
}
baos.write(classData, 0, bytesInBuffer);
buffer.rewind(); // rewind the buffer so that the next read starts at position 0 again
}
// Convert the output stream to an byte array and create an input stream
// based on this array
return baos.toByteArray();
} catch (IOException e) {
// Simply fall through to the finally block.
} finally {
if (fileContents != null) {
try {
fileContents.close();
} catch (IOException e) {
// Not really anything to do at this point. Fall through to return null.
}
}
}
return null;
}
public byte[] getfromUrl(String urlName) throws MalformedURLException, IOException {
int totalRead = 0;
URL url = new URL(urlName);
ReadableByteChannel rbc = Channels.newChannel(url.openStream());
ByteBuffer buf = ByteBuffer.allocateDirect(BUFFER_SIZE);
while (rbc.read(buf) != -1) {}
buf.flip();
totalRead = buf.limit();
LOG.debug("baixado: " + totalRead + " bytes de [" + urlName + "]");
byte[] b = new byte[totalRead];
buf.get(b, 0, totalRead);
rbc.close();
return b;
}