public String copy(String basePath, ImageRef source, String targetDirName) {
try {
ByteBuffer buffer = ByteBuffer.allocate(16 * 1024);
Resource res = source.eResource();
URI uri = source.eResource().getURI();
URI inPath = URI.createURI(source.getPath()).resolve(uri);
URI outPath =
URI.createURI(source.getPath())
.resolve(URI.createFileURI(new File(targetDirName).getAbsolutePath()));
ReadableByteChannel inChannel =
Channels.newChannel(res.getResourceSet().getURIConverter().createInputStream(inPath));
WritableByteChannel outChannel =
Channels.newChannel(res.getResourceSet().getURIConverter().createOutputStream(outPath));
while (inChannel.read(buffer) != -1) {
buffer.flip();
outChannel.write(buffer);
buffer.compact();
}
buffer.flip();
while (buffer.hasRemaining()) {
outChannel.write(buffer);
}
outChannel.close();
return outPath.toFileString();
} catch (IOException e) {
// TODO Auto-generated catch block
logger.error(e);
return null;
}
}
public void writeToChannel(WritableByteChannel channel) throws IOException {
channel.write(ByteBuffer.wrap(new byte[] {version}));
channel.write(ByteBuffer.wrap(Ints.toByteArray(totalSize)));
channel.write(ByteBuffer.wrap(Ints.toByteArray(sizePer)));
channel.write(ByteBuffer.wrap(new byte[] {compression.getId()}));
baseLongBuffers.writeToChannel(channel);
}
/**
* Helper Method to Copy from one Byte Channel to another.
*
* @param from
* @param to
* @throws IOException
*/
protected static void channelCopy(ReadableByteChannel from, WritableByteChannel to)
throws IOException {
ByteBuffer buffer = ByteBuffer.allocateDirect(16 * 1024);
try {
// Read
while (from.read(buffer) != -1) {
buffer.flip();
to.write(buffer);
buffer.compact();
}
// Flip the Buffer
buffer.flip();
// Write
while (buffer.hasRemaining()) {
to.write(buffer);
} // End of While Loop
} finally {
// Handle In Channel Closure
if (from != null) {
try {
from.close();
} catch (Exception ex) {
// No handling required
}
}
// Handle Out Channel Closure
if (to != null) {
try {
to.close();
} catch (Exception ex) {
// No handling required
}
}
} // End of Finally
}
/**
* Reads up to {@code count} bytes from this channel's file starting at {@code position} and
* writes them to {@code target}. No bytes are transferred if {@code position} is larger than the
* size of this channel's file. Less than {@code count} bytes are transferred if there less bytes
* available from this channel's file or if the target channel is non-blocking and has less than
* {@code count} bytes free in its input buffer.
*
* <p>Note that this channel's position is not modified.
*
* @param position the non-negative position to begin.
* @param count the non-negative number of bytes to transfer.
* @param target the target channel to write to.
* @return the number of bytes that were transferred.
* @throws IllegalArgumentException if the parameters are invalid.
* @throws NonReadableChannelException if this channel is not readable.
* @throws NonWritableChannelException if the target channel is not writable.
* @throws ClosedChannelException if either channel has already been closed.
* @throws AsynchronousCloseException if either channel is closed by other threads during this
* operation.
* @throws ClosedByInterruptException if the thread is interrupted during this operation.
* @throws IOException if any I/O error occurs.
*/
public long transferTo(long position, long count, WritableByteChannel target) throws IOException {
checkOpen();
if (!target.isOpen()) {
throw new ClosedChannelException();
}
checkReadable();
if (target instanceof IOCipherFileChannel) {
((IOCipherFileChannel) target).checkWritable();
}
if (position < 0 || count < 0) {
throw new IllegalArgumentException("position=" + position + " count=" + count);
}
if (count == 0 || position >= size()) {
return 0;
}
count = Math.min(count, size() - position);
try {
ByteBuffer buffer = ByteBuffer.allocate((int) count);
read(buffer, position);
buffer.flip();
return target.write(buffer);
} finally {
// TODO determine whether we have memory leaking or perf issues here #567
}
}
@Override
public void getBox(WritableByteChannel writableByteChannel) throws IOException {
writableByteChannel.write(getHeader());
ByteBuffer byteBuffer = ByteBuffer.allocate(52 + (fontName != null ? fontName.length() : 0));
byteBuffer.position(6);
IsoTypeWriter.writeUInt16(byteBuffer, dataReferenceIndex);
byteBuffer.putInt(displayFlags);
byteBuffer.putInt(textJustification);
IsoTypeWriter.writeUInt16(byteBuffer, backgroundR);
IsoTypeWriter.writeUInt16(byteBuffer, backgroundG);
IsoTypeWriter.writeUInt16(byteBuffer, backgroundB);
IsoTypeWriter.writeUInt64(byteBuffer, defaultTextBox);
IsoTypeWriter.writeUInt64(byteBuffer, reserved1);
byteBuffer.putShort(fontNumber);
byteBuffer.putShort(fontFace);
byteBuffer.put(reserved2);
byteBuffer.putShort(reserved3);
IsoTypeWriter.writeUInt16(byteBuffer, foregroundR);
IsoTypeWriter.writeUInt16(byteBuffer, foregroundG);
IsoTypeWriter.writeUInt16(byteBuffer, foregroundB);
if (fontName != null) {
IsoTypeWriter.writeUInt8(byteBuffer, fontName.length());
byteBuffer.put(fontName.getBytes());
}
writableByteChannel.write((ByteBuffer) byteBuffer.rewind());
// writeContainer(writableByteChannel); there are no child boxes!?
}
public void writeTo(Object obj, String mime, OutputStream os) throws IOException {
if (obj instanceof InputStream) {
InputStream is = (InputStream) obj;
byte[] buffer = new byte[BUFFER_SIZE];
int len;
while ((len = is.read(buffer)) != -1) os.write(buffer, 0, len);
os.flush();
} else if (obj instanceof FileRegionDataSource) {
FileRegionDataSource frds = (FileRegionDataSource) obj;
WritableByteChannel wbc = Channels.newChannel(os);
wbc.write(frds.getByteBuffer());
os.flush();
} else if (obj instanceof EmptyDataSource) {
/*
EmptyDataSource eds = (EmptyDataSource)obj;
InputStream is = (InputStream) eds.getInputStream();
byte[] buffer = new byte[BUFFER_SIZE];
int len;
while ((len = is.read(buffer)) != -1)
os.write(buffer, 0, len);
os.flush();
*/
} else if (obj instanceof String) {
byte[] bytes = ((String) obj).getBytes();
os.write(bytes);
os.flush();
} else {
throw new IOException("Unsupported class type - " + obj.getClass().getName());
}
}
public static long stream(InputStream input, OutputStream output) throws IOException {
ReadableByteChannel inputChannel = null;
WritableByteChannel outputChannel = null;
try {
inputChannel = Channels.newChannel(input);
outputChannel = Channels.newChannel(output);
ByteBuffer buffer = ByteBuffer.allocate(10240);
long size = 0;
while (inputChannel.read(buffer) != -1) {
buffer.flip();
size += outputChannel.write(buffer);
buffer.clear();
}
return size;
} finally {
if (outputChannel != null)
try {
outputChannel.close();
} catch (IOException ignore) {
/**/
}
if (inputChannel != null)
try {
inputChannel.close();
} catch (IOException ignore) {
/**/
}
}
}
/**
* Copy the readable byte channel to the writable byte channel
*
* @param inChannel the readable byte channel
* @param outChannel the writable byte channel
* @throws IOException if an IOException occurs.
*/
public static void copy(ReadableByteChannel inChannel, WritableByteChannel outChannel)
throws IOException {
if (inChannel instanceof FileChannel) {
((FileChannel) inChannel).transferTo(0, ((FileChannel) inChannel).size(), outChannel);
} else {
final ByteBuffer buffer = ByteBuffer.allocateDirect(16 * 1024);
try {
while (inChannel.read(buffer) != -1) {
// prepare the buffer to be drained
buffer.flip();
// write to the channel, may block
outChannel.write(buffer);
// If partial transfer, shift remainder down
// If buffer is empty, same as doing clear()
buffer.compact();
}
// EOF will leave buffer in fill state
buffer.flip();
// make sure the buffer is fully drained.
while (buffer.hasRemaining()) {
outChannel.write(buffer);
}
} finally {
IOUtils.close(inChannel);
IOUtils.close(outChannel);
}
}
}
public static void main(String[] argv) throws IOException {
ReadableByteChannel source = Channels.newChannel(System.in);
WritableByteChannel dest = Channels.newChannel(System.out);
channelCopy2(source, dest);
source.close();
dest.close();
}
/** Writes String contents to channel and closes it. */
public static void writeStringContents(String contents, WritableByteChannel channel)
throws IOException {
// TODO Checks if a supplier would be nice
try {
ByteBuffer buffer = ByteBuffer.wrap(contents.getBytes(StandardCharsets.UTF_8));
channel.write(buffer);
} finally {
channel.close();
}
}
/**
* Transfers data from the original message to the channel, encrypting it in the process.
*
* <p>This method also breaks down the original message into smaller chunks when needed. This is
* done to keep memory usage under control. This avoids having to copy the whole message data
* into memory at once, and can avoid ballooning memory usage when transferring large messages
* such as shuffle blocks.
*
* <p>The {@link #transfered()} counter also behaves a little funny, in that it won't go forward
* until a whole chunk has been written. This is done because the code can't use the actual
* number of bytes written to the channel as the transferred count (see {@link #count()}).
* Instead, once an encrypted chunk is written to the output (including its header), the size of
* the original block will be added to the {@link #transfered()} amount.
*/
@Override
public long transferTo(final WritableByteChannel target, final long position)
throws IOException {
Preconditions.checkArgument(position == transfered(), "Invalid position.");
long reportedWritten = 0L;
long actuallyWritten = 0L;
do {
if (currentChunk == null) {
nextChunk();
}
if (currentHeader.readableBytes() > 0) {
int bytesWritten = target.write(currentHeader.nioBuffer());
currentHeader.skipBytes(bytesWritten);
actuallyWritten += bytesWritten;
if (currentHeader.readableBytes() > 0) {
// Break out of loop if there are still header bytes left to write.
break;
}
}
actuallyWritten += target.write(currentChunk);
if (!currentChunk.hasRemaining()) {
// Only update the count of written bytes once a full chunk has been written.
// See method javadoc.
long chunkBytesRemaining = unencryptedChunkSize - currentReportedBytes;
reportedWritten += chunkBytesRemaining;
transferred += chunkBytesRemaining;
currentHeader.release();
currentHeader = null;
currentChunk = null;
currentChunkSize = 0;
currentReportedBytes = 0;
}
} while (currentChunk == null && transfered() + reportedWritten < count());
// Returning 0 triggers a backoff mechanism in netty which may harm performance. Instead,
// we return 1 until we can (i.e. until the reported count would actually match the size
// of the current chunk), at which point we resort to returning 0 so that the counts still
// match, at the cost of some performance. That situation should be rare, though.
if (reportedWritten != 0L) {
return reportedWritten;
}
if (actuallyWritten > 0 && currentReportedBytes < currentChunkSize - 1) {
transferred += 1L;
currentReportedBytes += 1L;
return 1L;
}
return 0L;
}
public static void writeFile(File file, byte[] bytes) throws IOException {
java.io.FileOutputStream f = new java.io.FileOutputStream(file);
final WritableByteChannel outputChannel = Channels.newChannel(f);
final ReadableByteChannel inputChannel = Channels.newChannel(new ByteArrayInputStream(bytes));
try {
fastChannelCopy(inputChannel, outputChannel);
// closing the channels
} finally {
inputChannel.close();
outputChannel.close();
}
}
public static void downloadFromMaster(Map conf, String file, String localFile)
throws IOException, TException {
NimbusClient client = NimbusClient.getConfiguredClient(conf);
String id = client.getClient().beginFileDownload(file);
WritableByteChannel out = Channels.newChannel(new FileOutputStream(localFile));
while (true) {
ByteBuffer chunk = client.getClient().downloadChunk(id);
int written = out.write(chunk);
if (written == 0) break;
}
out.close();
}
public static void copyStreams(final InputStream input, final OutputStream output)
throws IOException {
final ReadableByteChannel inputChannel = Channels.newChannel(input);
final WritableByteChannel outputChannel = Channels.newChannel(output);
try {
// copy the channels
fastChannelCopy(inputChannel, outputChannel);
// closing the channels
} finally {
inputChannel.close();
outputChannel.close();
}
}
/**
* Writes part of a byte array to a {@link java.nio.channels.WritableByteChannel}
*
* @param source byte array to copy from
* @param offset start copying from this offset
* @param length how many bytes to copy
* @param channel target WritableByteChannel
*/
public static void writeToChannel(
byte[] source, int offset, int length, WritableByteChannel channel) throws IOException {
int toWrite = Math.min(length, WRITE_CHUNK_SIZE);
ByteBuffer buffer = ByteBuffer.wrap(source, offset, toWrite);
int written = channel.write(buffer);
length -= written;
while (length > 0) {
toWrite = Math.min(length, WRITE_CHUNK_SIZE);
buffer.limit(buffer.position() + toWrite);
written = channel.write(buffer);
length -= written;
}
assert length == 0 : "wrote more then expected bytes (length=" + length + ")";
}
public static void main(String[] args) throws IOException {
FileInputStream fin = new FileInputStream(args[0]);
GZIPInputStream gzin = new GZIPInputStream(fin);
ReadableByteChannel in = Channels.newChannel(gzin);
WritableByteChannel out = Channels.newChannel(System.out);
ByteBuffer buffer = ByteBuffer.allocate(65536);
while (in.read(buffer) != -1) {
buffer.flip();
out.write(buffer);
buffer.clear();
}
}
private static void download(NimbusClient client, String file, String localFile)
throws IOException, TException, AuthorizationException {
WritableByteChannel out = Channels.newChannel(new FileOutputStream(localFile));
try {
String id = client.getClient().beginFileDownload(file);
while (true) {
ByteBuffer chunk = client.getClient().downloadChunk(id);
int written = out.write(chunk);
if (written == 0) break;
}
} finally {
out.close();
}
}
private void flushWriteBuffer() throws IOException {
while (!outBuffer.hasRemaining()) {
outBuffer.flip();
out.write(outBuffer);
outBuffer.compact();
}
}
public static void fastCopy(final ReadableByteChannel src, final WritableByteChannel dest)
throws IOException {
final ByteBuffer buffer = ByteBuffer.allocateDirect(16 * 1024);
while (src.read(buffer) != -1) {
buffer.flip();
dest.write(buffer);
buffer.compact();
}
buffer.flip();
while (buffer.hasRemaining()) {
dest.write(buffer);
}
}
/**
* This is a wrapper around {@link WritableByteChannel#write(ByteBuffer)}. If the amount of data
* is large, it writes to channel in smaller chunks. This is to avoid jdk from creating many
* direct buffers as the size of buffer increases. This also minimizes extra copies in NIO layer
* as a result of multiple write operations required to write a large buffer.
*
* @see WritableByteChannel#write(ByteBuffer)
*/
private static int channelWrite(WritableByteChannel channel, ByteBuffer buffer)
throws IOException {
return (buffer.remaining() <= NIO_BUFFER_LIMIT)
? channel.write(buffer)
: channelIO(null, channel, buffer);
}
/**
* Copy until boundary reached.
*
* @param target Output file channel
* @param boundary Boundary
* @throws IOException If fails
* @checkstyle ExecutableStatementCountCheck (2 lines)
*/
private void copy(final WritableByteChannel target, final byte[] boundary) throws IOException {
int match = 0;
boolean cont = true;
while (cont) {
if (!this.buffer.hasRemaining()) {
this.buffer.clear();
for (int idx = 0; idx < match; ++idx) {
this.buffer.put(boundary[idx]);
}
match = 0;
if (this.body.read(this.buffer) == -1) {
break;
}
this.buffer.flip();
}
final ByteBuffer btarget = this.buffer.slice();
final int offset = this.buffer.position();
btarget.limit(0);
while (this.buffer.hasRemaining()) {
final byte data = this.buffer.get();
if (data == boundary[match]) {
++match;
if (match == boundary.length) {
cont = false;
break;
}
} else {
match = 0;
btarget.limit(this.buffer.position() - offset);
}
}
target.write(btarget);
}
}
/**
* Helper for {@link #channelRead(ReadableByteChannel, ByteBuffer)} and {@link
* #channelWrite(WritableByteChannel, ByteBuffer)}. Only one of readCh or writeCh should be
* non-null.
*
* @see #channelRead(ReadableByteChannel, ByteBuffer)
* @see #channelWrite(WritableByteChannel, ByteBuffer)
*/
private static int channelIO(
ReadableByteChannel readCh, WritableByteChannel writeCh, ByteBuffer buf) throws IOException {
int originalLimit = buf.limit();
int initialRemaining = buf.remaining();
int ret = 0;
while (buf.remaining() > 0) {
try {
int ioSize = Math.min(buf.remaining(), NIO_BUFFER_LIMIT);
buf.limit(buf.position() + ioSize);
ret = (readCh == null) ? writeCh.write(buf) : readCh.read(buf);
if (ret < ioSize) {
break;
}
} finally {
buf.limit(originalLimit);
}
}
int nBytes = initialRemaining - buf.remaining();
return (nBytes > 0) ? nBytes : ret;
}
/**
* Platform-independent channel-to-channel transfer method. Uses regular {@code read} and {@code
* write} operations to move bytes from the {@code source} channel to the {@code sink} channel.
* After this call, the {@code throughBuffer} should be checked for remaining bytes; if there are
* any, they should be written to the {@code sink} channel before proceeding. This method may be
* used with NIO channels, XNIO channels, or a combination of the two.
*
* <p>If either or both of the given channels are blocking channels, then this method may block.
*
* @param source the source channel to read bytes from
* @param count the number of bytes to transfer (must be >= {@code 0L})
* @param throughBuffer the buffer to transfer through (must not be {@code null})
* @param sink the sink channel to write bytes to
* @return the number of bytes actually transferred (possibly 0)
* @throws IOException if an I/O error occurs during the transfer of bytes
*/
public static long transfer(
final ReadableByteChannel source,
final long count,
final ByteBuffer throughBuffer,
final WritableByteChannel sink)
throws IOException {
long res;
long total = 0L;
throughBuffer.limit(0);
while (total < count) {
throughBuffer.compact();
try {
if (count - total < (long) throughBuffer.remaining()) {
throughBuffer.limit((int) (count - total));
}
res = source.read(throughBuffer);
if (res <= 0) {
return total == 0L ? res : total;
}
} finally {
throughBuffer.flip();
}
res = sink.write(throughBuffer);
if (res == 0) {
return total;
}
total += res;
}
return total;
}
public long writeTo(WritableByteChannel channel, long maxBps) throws IOException {
// format:
// <format_version> <sig_len> <sig_data> { <idx_file_name_len> <idx_file_name> <idx_file_len>
// <idx_file_data> }...
long amount = 0;
// format version
amount += ChannelUtil.writeInt(channel, 1);
// index signature
ByteArrayOutputStream baos = new ByteArrayOutputStream();
_sig.save(baos);
byte[] sigBytes = baos.toByteArray();
amount += ChannelUtil.writeLong(channel, (long) sigBytes.length); // data length
amount += channel.write(ByteBuffer.wrap(sigBytes)); // data
// index files
Collection<String> fileNames = _snapshot.getFileNames();
amount += ChannelUtil.writeInt(channel, fileNames.size()); // number of files
for (String fileName : fileNames) {
amount += ChannelUtil.writeString(channel, fileName);
amount += _dirMgr.transferFromFileToChannel(fileName, channel, maxBps);
}
return amount;
}
public static void copy(InputStream in, OutputStream out) throws IOException {
ReadableByteChannel source = Channels.newChannel(in);
WritableByteChannel target = Channels.newChannel(out);
ByteBuffer buffer = ByteBuffer.allocate(16 * 1024);
while (source.read(buffer) != -1) {
buffer.flip(); // Prepare the buffer to be drained
while (buffer.hasRemaining()) {
target.write(buffer);
}
buffer.clear(); // Empty buffer to get ready for filling
}
source.close();
target.close();
}
private static void channelCopy1(ReadableByteChannel src, WritableByteChannel dest)
throws IOException {
ByteBuffer buffer = ByteBuffer.allocateDirect(16 * 1024);
while (src.read(buffer) != -1) {
buffer.flip();
dest.write(buffer);
buffer.compact();
}
buffer.flip();
while (buffer.hasRemaining()) {
dest.write(buffer);
}
}
/**
* Read data from a top level Variable and send data to a WritableByteChannel.
*
* @param v2 Variable
* @param section wanted section of data of Variable. The section list is a list of ucar.ma2.Range
* which define the requested data subset.
* @param channel WritableByteChannel object - channel that can write bytes.
* @return the number of bytes written, possibly zero.
*/
public long readToByteChannel11(
ucar.nc2.Variable v2, Section section, WritableByteChannel channel)
throws java.io.IOException, ucar.ma2.InvalidRangeException {
Array data = readData(v2, section);
float[] ftdata = new float[(int) data.getSize()];
byte[] bytedata = new byte[(int) data.getSize() * 4];
IndexIterator iter = data.getIndexIterator();
int i = 0;
ByteBuffer buffer = ByteBuffer.allocateDirect(bytedata.length);
while (iter.hasNext()) {
ftdata[i] = iter.getFloatNext();
bytedata[i] = new Float(ftdata[i]).byteValue();
buffer.put(bytedata[i]);
i++;
}
buffer = ByteBuffer.wrap(bytedata);
// write the bytes to the channel
int count = channel.write(buffer);
System.out.println("COUNT=" + count);
// check if all bytes where written
if (buffer.hasRemaining()) {
// if not all bytes were written, move the unwritten bytes to the beginning and
// set position just after the last unwritten byte
buffer.compact();
} else {
buffer.clear();
}
return (long) count;
}
private void encodeCommand(@Nonnull AbstractCommand cmd) throws IOException {
if (cmd.getId() != CommandList.CMD_KEEPALIVE) {
log.debug("SND: {}", cmd);
}
buffer.clear();
buffer.put((byte) cmd.getId());
buffer.put((byte) (cmd.getId() ^ COMMAND_XOR_MASK));
// keep some space for the length and the CRC
int headerLenCRC = buffer.position();
buffer.putShort((short) 0);
buffer.putShort((short) 0);
int startOfCmd = buffer.position();
// encode command into net protocol
cmd.encode(this);
int length = buffer.position() - startOfCmd;
buffer.flip();
buffer.position(startOfCmd);
int crc = NetComm.getCRC(buffer, length);
buffer.position(headerLenCRC);
buffer.putShort((short) length);
buffer.putShort((short) crc);
buffer.position(0);
if (NetComm.isDumpingActive()) {
NetComm.dump("snd => ", buffer);
buffer.flip();
}
outChannel.write(buffer);
}
public static int writeToChannel(WritableByteChannel channel, ByteBuffer buffer)
throws NetIOException {
try {
return channel.write(buffer);
} catch (IOException e) {
return -1;
}
}
@Override
public long transferTo(WritableByteChannel target, long position) throws IOException {
if (this.byteBufferHeader.hasRemaining()) {
transfered += target.write(this.byteBufferHeader);
return transfered;
} else {
List<ByteBuffer> messageBufferList = this.queryMessageResult.getMessageBufferList();
for (ByteBuffer bb : messageBufferList) {
if (bb.hasRemaining()) {
transfered += target.write(bb);
return transfered;
}
}
}
return 0;
}
