public static long stream(InputStream input, OutputStream output) throws IOException {
ReadableByteChannel inputChannel = null;
WritableByteChannel outputChannel = null;
try {
inputChannel = Channels.newChannel(input);
outputChannel = Channels.newChannel(output);
ByteBuffer buffer = ByteBuffer.allocate(10240);
long size = 0;
while (inputChannel.read(buffer) != -1) {
buffer.flip();
size += outputChannel.write(buffer);
buffer.clear();
}
return size;
} finally {
if (outputChannel != null)
try {
outputChannel.close();
} catch (IOException ignore) {
/**/
}
if (inputChannel != null)
try {
inputChannel.close();
} catch (IOException ignore) {
/**/
}
}
}
/**
* transfer Stream
*
* @param ins
* @param targetChannel
*/
private static void transferStream(InputStream ins, FileChannel targetChannel) {
ByteBuffer byteBuffer = ByteBuffer.allocate(1024 * 10);
ReadableByteChannel rbcInst = Channels.newChannel(ins);
try {
while (-1 != (rbcInst.read(byteBuffer))) {
byteBuffer.flip();
targetChannel.write(byteBuffer);
byteBuffer.clear();
}
} catch (IOException ioe) {
ioe.printStackTrace();
} finally {
if (null != rbcInst) {
try {
rbcInst.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (null != targetChannel) {
try {
targetChannel.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
@org.junit.Test
public void testMapper() throws Exception {
final ArcFileReader reader = new ArcFileReader();
Thread thread =
new Thread(
new Runnable() {
public void run() {
try {
while (reader.hasMoreItems()) {
ArcFileItem item = new ArcFileItem();
reader.getNextItem(item);
map(new Text(item.getUri()), item, null, null);
}
LOG.info("NO MORE ITEMS... BYE");
} catch (IOException e) {
LOG.error(StringUtils.stringifyException(e));
}
}
});
// run the thread ...
thread.start();
File file = new File("/Users/rana/Downloads/1213886083018_0.arc.gz");
ReadableByteChannel channel = Channels.newChannel(new FileInputStream(file));
try {
int totalBytesRead = 0;
for (; ; ) {
ByteBuffer buffer = ByteBuffer.allocate(ArcFileReader.DEFAULT_BLOCK_SIZE);
int bytesRead = channel.read(buffer);
LOG.info("Read " + bytesRead + " From File");
if (bytesRead == -1) {
reader.finished();
break;
} else {
buffer.flip();
totalBytesRead += buffer.remaining();
reader.available(buffer);
}
}
} finally {
channel.close();
}
// now wait for thread to die ...
LOG.info("Done Reading File.... Waiting for ArcFileThread to DIE");
thread.join();
LOG.info("Done Reading File.... ArcFileThread to DIED");
}
/**
* Helper Method to Copy from one Byte Channel to another.
*
* @param from
* @param to
* @throws IOException
*/
protected static void channelCopy(ReadableByteChannel from, WritableByteChannel to)
throws IOException {
ByteBuffer buffer = ByteBuffer.allocateDirect(16 * 1024);
try {
// Read
while (from.read(buffer) != -1) {
buffer.flip();
to.write(buffer);
buffer.compact();
}
// Flip the Buffer
buffer.flip();
// Write
while (buffer.hasRemaining()) {
to.write(buffer);
} // End of While Loop
} finally {
// Handle In Channel Closure
if (from != null) {
try {
from.close();
} catch (Exception ex) {
// No handling required
}
}
// Handle Out Channel Closure
if (to != null) {
try {
to.close();
} catch (Exception ex) {
// No handling required
}
}
} // End of Finally
}
public String copy(String basePath, ImageRef source, String targetDirName) {
try {
ByteBuffer buffer = ByteBuffer.allocate(16 * 1024);
Resource res = source.eResource();
URI uri = source.eResource().getURI();
URI inPath = URI.createURI(source.getPath()).resolve(uri);
URI outPath =
URI.createURI(source.getPath())
.resolve(URI.createFileURI(new File(targetDirName).getAbsolutePath()));
ReadableByteChannel inChannel =
Channels.newChannel(res.getResourceSet().getURIConverter().createInputStream(inPath));
WritableByteChannel outChannel =
Channels.newChannel(res.getResourceSet().getURIConverter().createOutputStream(outPath));
while (inChannel.read(buffer) != -1) {
buffer.flip();
outChannel.write(buffer);
buffer.compact();
}
buffer.flip();
while (buffer.hasRemaining()) {
outChannel.write(buffer);
}
outChannel.close();
return outPath.toFileString();
} catch (IOException e) {
// TODO Auto-generated catch block
logger.error(e);
return null;
}
}
static int safeRead(ReadableByteChannel channel, ByteBuffer dst) throws IOException {
int read = -1;
try {
// Read data from the Channel
read = channel.read(dst);
} catch (ClosedChannelException e) {
} catch (IOException e) {
switch ("" + e.getMessage()) {
case "null":
case "Connection reset by peer":
case "Broken pipe":
break;
default:
cntIoError++;
System.out.println("error count IoError: " + cntIoError);
}
channel.close();
e.printStackTrace();
} catch (CancelledKeyException e) {
cntCancelledKeyException++;
System.out.println("error count CancelledKey: " + cntCancelledKeyException);
channel.close();
}
return read;
}
public static void main(String[] argv) throws IOException {
ReadableByteChannel source = Channels.newChannel(System.in);
WritableByteChannel dest = Channels.newChannel(System.out);
channelCopy2(source, dest);
source.close();
dest.close();
}
/**
* Reads the given website with the URL {@code url} or downloads its
* contents and returns them as a {@link String}.
*
* @param url
* the URL
* @return the lines
*/
public static String readWebsite(String url) throws IOException
{
URL url1 = new URL(url);
ReadableByteChannel rbc = Channels.newChannel(url1.openStream());
ByteBuffer bytebuf = ByteBuffer.allocate(1024);
rbc.read(bytebuf);
return new String(bytebuf.array());
}
/**
* Refills the byte buffer.
*
* @param sourceChannel The byte channel to read from.
* @return The number of bytes read and added or -1 if end of the source channel reached.
* @throws IOException
*/
public int fill(ReadableByteChannel sourceChannel) throws IOException {
int result = 0;
if (sourceChannel.isOpen()) {
result = sourceChannel.read(getBytes());
}
return result;
}
/** Dispose of this <code>InputStreamImageDataProvider</code> and its resources. */
public synchronized void dispose() {
if (m_channel != null) {
final ReadableByteChannel temp = m_channel;
m_channel = null;
try {
temp.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
public static void writeFile(File file, byte[] bytes) throws IOException {
java.io.FileOutputStream f = new java.io.FileOutputStream(file);
final WritableByteChannel outputChannel = Channels.newChannel(f);
final ReadableByteChannel inputChannel = Channels.newChannel(new ByteArrayInputStream(bytes));
try {
fastChannelCopy(inputChannel, outputChannel);
// closing the channels
} finally {
inputChannel.close();
outputChannel.close();
}
}
public static void downloadFileFromURL(String urlString, File destination) {
try {
URL website = new URL(urlString);
ReadableByteChannel rbc;
rbc = Channels.newChannel(website.openStream());
FileOutputStream fos = new FileOutputStream(destination);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos.close();
rbc.close();
} catch (IOException e) {
e.printStackTrace();
}
}
public static void copyStreams(final InputStream input, final OutputStream output)
throws IOException {
final ReadableByteChannel inputChannel = Channels.newChannel(input);
final WritableByteChannel outputChannel = Channels.newChannel(output);
try {
// copy the channels
fastChannelCopy(inputChannel, outputChannel);
// closing the channels
} finally {
inputChannel.close();
outputChannel.close();
}
}
/**
* Perform a low-level read of the remaining number of bytes into the specified byte buffer. The
* incoming bytes will be used to fill the remaining space in the target byte buffer. This is
* equivalent to the read(2) POSIX function, and like that function it ignores read and write
* buffers defined elsewhere.
*
* @param buffer the java.nio.ByteBuffer in which to put the incoming bytes
* @return the number of bytes actually read
* @throws java.io.IOException if there is an exception during IO
* @throws org.jruby.util.io.BadDescriptorException if the associated channel is already closed
* @see java.nio.ByteBuffer
*/
public int read(ByteBuffer buffer) throws IOException, BadDescriptorException {
checkOpen();
// TODO: It would be nice to throw a better error for this
if (!isReadable()) {
throw new BadDescriptorException();
}
ReadableByteChannel readChannel = (ReadableByteChannel) channel;
int bytesRead = 0;
bytesRead = readChannel.read(buffer);
return bytesRead;
}
public static void main(String[] args) throws IOException {
FileInputStream fin = new FileInputStream(args[0]);
GZIPInputStream gzin = new GZIPInputStream(fin);
ReadableByteChannel in = Channels.newChannel(gzin);
WritableByteChannel out = Channels.newChannel(System.out);
ByteBuffer buffer = ByteBuffer.allocate(65536);
while (in.read(buffer) != -1) {
buffer.flip();
out.write(buffer);
buffer.clear();
}
}
/**
* Load the byte code for a class from a file based on the path and class name. Created: Mar 30,
* 2004
*
* @param fileLocator
* @return - the byte code for the class.
*/
private byte[] loadFileData(ProgramResourceLocator.File fileLocator) {
InputStream fileContents = null;
try {
fileContents =
resourceRepository.getContents(
fileLocator); // throws IOException if the file doesn't exist.
ReadableByteChannel channel = Channels.newChannel(fileContents);
// Note: Another way to do this is to determine the channel size, and allocate an array of the
// correct size.
// However, this involves additional file system access.
// Init the output buffer for holding the input stream
ByteArrayOutputStream baos = threadLocalByteArrayOutputStream.get();
baos.reset();
byte[] classData = threadLocalByteArray.get();
ByteBuffer buffer = ByteBuffer.wrap(classData);
// Read the input stream to a buffer and write the buffer to the output buffer
while (true) {
int bytesInBuffer = channel.read(buffer);
if (bytesInBuffer < 0) {
break;
}
baos.write(classData, 0, bytesInBuffer);
buffer.rewind(); // rewind the buffer so that the next read starts at position 0 again
}
// Convert the output stream to an byte array and create an input stream
// based on this array
return baos.toByteArray();
} catch (IOException e) {
// Simply fall through to the finally block.
} finally {
if (fileContents != null) {
try {
fileContents.close();
} catch (IOException e) {
// Not really anything to do at this point. Fall through to return null.
}
}
}
return null;
}
/**
* Downloads from an url and store the image.
*
* @param idImage
* @param url
*/
public void storeImage(String idImage, String url) {
try {
URL website = new URL(url);
Path pathToFile = Paths.get(getRoot());
ReadableByteChannel rbc;
rbc = Channels.newChannel(website.openStream());
FileOutputStream fos = new FileOutputStream(pathToFile.resolve(idImage).toFile());
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos.flush();
fos.close();
rbc.close();
} catch (IOException e) {
e.printStackTrace();
}
}
public byte[] getfromUrl(String urlName) throws MalformedURLException, IOException {
int totalRead = 0;
URL url = new URL(urlName);
ReadableByteChannel rbc = Channels.newChannel(url.openStream());
ByteBuffer buf = ByteBuffer.allocateDirect(BUFFER_SIZE);
while (rbc.read(buf) != -1) {}
buf.flip();
totalRead = buf.limit();
LOG.debug("baixado: " + totalRead + " bytes de [" + urlName + "]");
byte[] b = new byte[totalRead];
buf.get(b, 0, totalRead);
rbc.close();
return b;
}
public static void copy(InputStream in, OutputStream out) throws IOException {
ReadableByteChannel source = Channels.newChannel(in);
WritableByteChannel target = Channels.newChannel(out);
ByteBuffer buffer = ByteBuffer.allocate(16 * 1024);
while (source.read(buffer) != -1) {
buffer.flip(); // Prepare the buffer to be drained
while (buffer.hasRemaining()) {
target.write(buffer);
}
buffer.clear(); // Empty buffer to get ready for filling
}
source.close();
target.close();
}
/**
* Copy the readable byte channel to the writable byte channel
*
* @param inChannel the readable byte channel
* @param outChannel the writable byte channel
* @throws IOException if an IOException occurs.
*/
public static void copy(ReadableByteChannel inChannel, WritableByteChannel outChannel)
throws IOException {
if (inChannel instanceof FileChannel) {
((FileChannel) inChannel).transferTo(0, ((FileChannel) inChannel).size(), outChannel);
} else {
final ByteBuffer buffer = ByteBuffer.allocateDirect(16 * 1024);
try {
while (inChannel.read(buffer) != -1) {
// prepare the buffer to be drained
buffer.flip();
// write to the channel, may block
outChannel.write(buffer);
// If partial transfer, shift remainder down
// If buffer is empty, same as doing clear()
buffer.compact();
}
// EOF will leave buffer in fill state
buffer.flip();
// make sure the buffer is fully drained.
while (buffer.hasRemaining()) {
outChannel.write(buffer);
}
} finally {
IOUtils.close(inChannel);
IOUtils.close(outChannel);
}
}
}
public boolean onData(INonBlockingConnection connection)
throws IOException, BufferUnderflowException, MaxReadSizeExceededException {
connection.readStringByDelimiter("\r\n");
File file = QAUtil.createTestfile_400k();
RandomAccessFile raf = new RandomAccessFile(file, "r");
ReadableByteChannel in = raf.getChannel();
connection.transferFrom(in);
in.close();
raf.close();
file.delete();
return true;
}
/**
* Platform-independent channel-to-channel transfer method. Uses regular {@code read} and {@code
* write} operations to move bytes from the {@code source} channel to the {@code sink} channel.
* After this call, the {@code throughBuffer} should be checked for remaining bytes; if there are
* any, they should be written to the {@code sink} channel before proceeding. This method may be
* used with NIO channels, XNIO channels, or a combination of the two.
*
* <p>If either or both of the given channels are blocking channels, then this method may block.
*
* @param source the source channel to read bytes from
* @param count the number of bytes to transfer (must be >= {@code 0L})
* @param throughBuffer the buffer to transfer through (must not be {@code null})
* @param sink the sink channel to write bytes to
* @return the number of bytes actually transferred (possibly 0)
* @throws IOException if an I/O error occurs during the transfer of bytes
*/
public static long transfer(
final ReadableByteChannel source,
final long count,
final ByteBuffer throughBuffer,
final WritableByteChannel sink)
throws IOException {
long res;
long total = 0L;
throughBuffer.limit(0);
while (total < count) {
throughBuffer.compact();
try {
if (count - total < (long) throughBuffer.remaining()) {
throughBuffer.limit((int) (count - total));
}
res = source.read(throughBuffer);
if (res <= 0) {
return total == 0L ? res : total;
}
} finally {
throughBuffer.flip();
}
res = sink.write(throughBuffer);
if (res == 0) {
return total;
}
total += res;
}
return total;
}
/**
* Clean up any resources. Closes the channel.
*
* @throws IOException If errors occur while closing the channel.
*/
public void close() throws IOException {
// don't throw NPE on double close
if (channel == null) return;
try {
if (channel.isOpen()) {
channel.close();
streamLogger.close();
}
NIOUtilities.clean(buffer, useMemoryMappedBuffer);
} finally {
if (shxReader != null) shxReader.close();
}
shxReader = null;
channel = null;
header = null;
}
/**
* This is a wrapper around {@link ReadableByteChannel#read(ByteBuffer)}. If the amount of data is
* large, it writes to channel in smaller chunks. This is to avoid jdk from creating many direct
* buffers as the size of ByteBuffer increases. There should not be any performance degredation.
*
* @see ReadableByteChannel#read(ByteBuffer)
*/
private static int channelRead(ReadableByteChannel channel, ByteBuffer buffer)
throws IOException {
return (buffer.remaining() <= NIO_BUFFER_LIMIT)
? channel.read(buffer)
: channelIO(channel, null, buffer);
}
/**
* Helper for {@link #channelRead(ReadableByteChannel, ByteBuffer)} and {@link
* #channelWrite(WritableByteChannel, ByteBuffer)}. Only one of readCh or writeCh should be
* non-null.
*
* @see #channelRead(ReadableByteChannel, ByteBuffer)
* @see #channelWrite(WritableByteChannel, ByteBuffer)
*/
private static int channelIO(
ReadableByteChannel readCh, WritableByteChannel writeCh, ByteBuffer buf) throws IOException {
int originalLimit = buf.limit();
int initialRemaining = buf.remaining();
int ret = 0;
while (buf.remaining() > 0) {
try {
int ioSize = Math.min(buf.remaining(), NIO_BUFFER_LIMIT);
buf.limit(buf.position() + ioSize);
ret = (readCh == null) ? writeCh.write(buf) : readCh.read(buf);
if (ret < ioSize) {
break;
}
} finally {
buf.limit(originalLimit);
}
}
int nBytes = initialRemaining - buf.remaining();
return (nBytes > 0) ? nBytes : ret;
}
private void fetch(String res) throws IOException {
URL url = new URL(Configuration.composeres() + res);
HttpURLConnection con = (HttpURLConnection) url.openConnection();
con.setRequestMethod("GET");
con.setDoInput(true);
con.setConnectTimeout(5000);
con.setRequestProperty(
"User-Agent",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11");
ReadableByteChannel rbc = Channels.newChannel(con.getInputStream());
FileOutputStream fos = new FileOutputStream(Configuration.STORAGE_DIR + File.separator + res);
fos.getChannel().transferFrom(rbc, 0, 1 << 24);
fos.close();
rbc.close();
}
private int doRead(ReadableByteChannel channel, ByteBuffer byteBuffer) throws IOException {
int readBytes = 0;
int lastReadBytes = -1;
try {
do {
lastReadBytes = channel.read(byteBuffer);
if (lastReadBytes > 0) {
readBytes += lastReadBytes;
} else if (lastReadBytes == -1 && readBytes == 0) {
readBytes = -1;
}
} while (lastReadBytes > 0 && byteBuffer.hasRemaining());
} catch (IOException ex) {
lastReadBytes = -1;
throw ex;
} finally {
if (lastReadBytes == -1 || readBytes == -1) {
SelectionKeyHandler skh = selectorHandler.getSelectionKeyHandler();
if (skh instanceof BaseSelectionKeyHandler) {
((BaseSelectionKeyHandler) skh)
.notifyRemotlyClose(
((SelectableChannel) channel).keyFor(selectorHandler.getSelector()));
}
}
}
return readBytes;
}
public void write(
String path, ReadableByteChannel data, ByteBuffer temporaryBuffer, boolean hasData)
throws IOException {
try {
File file = new File(basePath, path);
RandomAccessFile randomAccessFile = null;
try {
file.getParentFile().mkdirs();
randomAccessFile = new RandomAccessFile(file, "rw");
if (hasData) {
FileChannel channel = randomAccessFile.getChannel();
while (data.read(temporaryBuffer) >= 0) {
temporaryBuffer.flip();
channel.write(temporaryBuffer);
temporaryBuffer.clear();
}
}
} finally {
if (randomAccessFile != null) {
randomAccessFile.close();
}
}
} catch (Throwable t) {
t.printStackTrace();
throw new IOException(t);
}
}
@Test(expected = IOException.class)
public void testTruncatedData() throws IOException {
// with BZ2File(self.filename) as f:
// self.assertRaises(EOFError, f.read)
System.out.println("Attempt to read the whole thing in, should throw ...");
ByteBuffer buffer = ByteBuffer.allocate(8192);
bz2Channel.read(buffer);
}
public static int readToBuffer(ReadableByteChannel channel, ByteBuffer buffer)
throws NetIOException {
try {
return channel.read(buffer);
} catch (IOException e) {
return -1;
}
}