示例#1
0
  public static long stream(InputStream input, OutputStream output) throws IOException {
    ReadableByteChannel inputChannel = null;
    WritableByteChannel outputChannel = null;

    try {
      inputChannel = Channels.newChannel(input);
      outputChannel = Channels.newChannel(output);
      ByteBuffer buffer = ByteBuffer.allocate(10240);
      long size = 0;

      while (inputChannel.read(buffer) != -1) {
        buffer.flip();
        size += outputChannel.write(buffer);
        buffer.clear();
      }

      return size;
    } finally {
      if (outputChannel != null)
        try {
          outputChannel.close();
        } catch (IOException ignore) {
          /**/
        }
      if (inputChannel != null)
        try {
          inputChannel.close();
        } catch (IOException ignore) {
          /**/
        }
    }
  }
示例#2
0
 /**
  * transfer Stream
  *
  * @param ins
  * @param targetChannel
  */
 private static void transferStream(InputStream ins, FileChannel targetChannel) {
   ByteBuffer byteBuffer = ByteBuffer.allocate(1024 * 10);
   ReadableByteChannel rbcInst = Channels.newChannel(ins);
   try {
     while (-1 != (rbcInst.read(byteBuffer))) {
       byteBuffer.flip();
       targetChannel.write(byteBuffer);
       byteBuffer.clear();
     }
   } catch (IOException ioe) {
     ioe.printStackTrace();
   } finally {
     if (null != rbcInst) {
       try {
         rbcInst.close();
       } catch (IOException e) {
         e.printStackTrace();
       }
     }
     if (null != targetChannel) {
       try {
         targetChannel.close();
       } catch (IOException e) {
         e.printStackTrace();
       }
     }
   }
 }
  @org.junit.Test
  public void testMapper() throws Exception {

    final ArcFileReader reader = new ArcFileReader();

    Thread thread =
        new Thread(
            new Runnable() {

              public void run() {
                try {

                  while (reader.hasMoreItems()) {
                    ArcFileItem item = new ArcFileItem();

                    reader.getNextItem(item);

                    map(new Text(item.getUri()), item, null, null);
                  }
                  LOG.info("NO MORE ITEMS... BYE");
                } catch (IOException e) {
                  LOG.error(StringUtils.stringifyException(e));
                }
              }
            });

    // run the thread ...
    thread.start();

    File file = new File("/Users/rana/Downloads/1213886083018_0.arc.gz");
    ReadableByteChannel channel = Channels.newChannel(new FileInputStream(file));

    try {

      int totalBytesRead = 0;
      for (; ; ) {

        ByteBuffer buffer = ByteBuffer.allocate(ArcFileReader.DEFAULT_BLOCK_SIZE);

        int bytesRead = channel.read(buffer);
        LOG.info("Read " + bytesRead + " From File");

        if (bytesRead == -1) {
          reader.finished();
          break;
        } else {
          buffer.flip();
          totalBytesRead += buffer.remaining();
          reader.available(buffer);
        }
      }
    } finally {
      channel.close();
    }

    // now wait for thread to die ...
    LOG.info("Done Reading File.... Waiting for ArcFileThread to DIE");
    thread.join();
    LOG.info("Done Reading File.... ArcFileThread to DIED");
  }
示例#4
0
 /**
  * Helper Method to Copy from one Byte Channel to another.
  *
  * @param from
  * @param to
  * @throws IOException
  */
 protected static void channelCopy(ReadableByteChannel from, WritableByteChannel to)
     throws IOException {
   ByteBuffer buffer = ByteBuffer.allocateDirect(16 * 1024);
   try {
     // Read
     while (from.read(buffer) != -1) {
       buffer.flip();
       to.write(buffer);
       buffer.compact();
     }
     // Flip the Buffer
     buffer.flip();
     // Write
     while (buffer.hasRemaining()) {
       to.write(buffer);
     } // End of While Loop
   } finally {
     // Handle In Channel Closure
     if (from != null) {
       try {
         from.close();
       } catch (Exception ex) {
         // No handling required
       }
     }
     // Handle Out Channel Closure
     if (to != null) {
       try {
         to.close();
       } catch (Exception ex) {
         // No handling required
       }
     }
   } // End of Finally
 }
示例#5
0
 public String copy(String basePath, ImageRef source, String targetDirName) {
   try {
     ByteBuffer buffer = ByteBuffer.allocate(16 * 1024);
     Resource res = source.eResource();
     URI uri = source.eResource().getURI();
     URI inPath = URI.createURI(source.getPath()).resolve(uri);
     URI outPath =
         URI.createURI(source.getPath())
             .resolve(URI.createFileURI(new File(targetDirName).getAbsolutePath()));
     ReadableByteChannel inChannel =
         Channels.newChannel(res.getResourceSet().getURIConverter().createInputStream(inPath));
     WritableByteChannel outChannel =
         Channels.newChannel(res.getResourceSet().getURIConverter().createOutputStream(outPath));
     while (inChannel.read(buffer) != -1) {
       buffer.flip();
       outChannel.write(buffer);
       buffer.compact();
     }
     buffer.flip();
     while (buffer.hasRemaining()) {
       outChannel.write(buffer);
     }
     outChannel.close();
     return outPath.toFileString();
   } catch (IOException e) {
     // TODO Auto-generated catch block
     logger.error(e);
     return null;
   }
 }
  static int safeRead(ReadableByteChannel channel, ByteBuffer dst) throws IOException {
    int read = -1;
    try {
      // Read data from the Channel
      read = channel.read(dst);
    } catch (ClosedChannelException e) {

    } catch (IOException e) {
      switch ("" + e.getMessage()) {
        case "null":
        case "Connection reset by peer":
        case "Broken pipe":
          break;
        default:
          cntIoError++;
          System.out.println("error count IoError: " + cntIoError);
      }
      channel.close();
      e.printStackTrace();
    } catch (CancelledKeyException e) {
      cntCancelledKeyException++;
      System.out.println("error count CancelledKey: " + cntCancelledKeyException);
      channel.close();
    }
    return read;
  }
示例#7
0
  public static void main(String[] argv) throws IOException {
    ReadableByteChannel source = Channels.newChannel(System.in);
    WritableByteChannel dest = Channels.newChannel(System.out);

    channelCopy2(source, dest);
    source.close();
    dest.close();
  }
示例#8
0
	/**
	 * Reads the given website with the URL {@code url} or downloads its
	 * contents and returns them as a {@link String}.
	 * 
	 * @param url
	 *            the URL
	 * @return the lines
	 */
	public static String readWebsite(String url) throws IOException
	{
		URL url1 = new URL(url);
		ReadableByteChannel rbc = Channels.newChannel(url1.openStream());
		ByteBuffer bytebuf = ByteBuffer.allocate(1024);
		rbc.read(bytebuf);
		return new String(bytebuf.array());
	}
  /**
   * Refills the byte buffer.
   *
   * @param sourceChannel The byte channel to read from.
   * @return The number of bytes read and added or -1 if end of the source channel reached.
   * @throws IOException
   */
  public int fill(ReadableByteChannel sourceChannel) throws IOException {
    int result = 0;

    if (sourceChannel.isOpen()) {
      result = sourceChannel.read(getBytes());
    }

    return result;
  }
 /** Dispose of this <code>InputStreamImageDataProvider</code> and its resources. */
 public synchronized void dispose() {
   if (m_channel != null) {
     final ReadableByteChannel temp = m_channel;
     m_channel = null;
     try {
       temp.close();
     } catch (IOException e) {
       e.printStackTrace();
     }
   }
 }
示例#11
0
 public static void writeFile(File file, byte[] bytes) throws IOException {
   java.io.FileOutputStream f = new java.io.FileOutputStream(file);
   final WritableByteChannel outputChannel = Channels.newChannel(f);
   final ReadableByteChannel inputChannel = Channels.newChannel(new ByteArrayInputStream(bytes));
   try {
     fastChannelCopy(inputChannel, outputChannel);
     // closing the channels
   } finally {
     inputChannel.close();
     outputChannel.close();
   }
 }
 public static void downloadFileFromURL(String urlString, File destination) {
   try {
     URL website = new URL(urlString);
     ReadableByteChannel rbc;
     rbc = Channels.newChannel(website.openStream());
     FileOutputStream fos = new FileOutputStream(destination);
     fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
     fos.close();
     rbc.close();
   } catch (IOException e) {
     e.printStackTrace();
   }
 }
示例#13
0
 public static void copyStreams(final InputStream input, final OutputStream output)
     throws IOException {
   final ReadableByteChannel inputChannel = Channels.newChannel(input);
   final WritableByteChannel outputChannel = Channels.newChannel(output);
   try {
     // copy the channels
     fastChannelCopy(inputChannel, outputChannel);
     // closing the channels
   } finally {
     inputChannel.close();
     outputChannel.close();
   }
 }
  /**
   * Perform a low-level read of the remaining number of bytes into the specified byte buffer. The
   * incoming bytes will be used to fill the remaining space in the target byte buffer. This is
   * equivalent to the read(2) POSIX function, and like that function it ignores read and write
   * buffers defined elsewhere.
   *
   * @param buffer the java.nio.ByteBuffer in which to put the incoming bytes
   * @return the number of bytes actually read
   * @throws java.io.IOException if there is an exception during IO
   * @throws org.jruby.util.io.BadDescriptorException if the associated channel is already closed
   * @see java.nio.ByteBuffer
   */
  public int read(ByteBuffer buffer) throws IOException, BadDescriptorException {
    checkOpen();

    // TODO: It would be nice to throw a better error for this
    if (!isReadable()) {
      throw new BadDescriptorException();
    }
    ReadableByteChannel readChannel = (ReadableByteChannel) channel;
    int bytesRead = 0;
    bytesRead = readChannel.read(buffer);

    return bytesRead;
  }
示例#15
0
  public static void main(String[] args) throws IOException {

    FileInputStream fin = new FileInputStream(args[0]);
    GZIPInputStream gzin = new GZIPInputStream(fin);
    ReadableByteChannel in = Channels.newChannel(gzin);

    WritableByteChannel out = Channels.newChannel(System.out);
    ByteBuffer buffer = ByteBuffer.allocate(65536);
    while (in.read(buffer) != -1) {
      buffer.flip();
      out.write(buffer);
      buffer.clear();
    }
  }
示例#16
0
  /**
   * Load the byte code for a class from a file based on the path and class name. Created: Mar 30,
   * 2004
   *
   * @param fileLocator
   * @return - the byte code for the class.
   */
  private byte[] loadFileData(ProgramResourceLocator.File fileLocator) {

    InputStream fileContents = null;
    try {
      fileContents =
          resourceRepository.getContents(
              fileLocator); // throws IOException if the file doesn't exist.
      ReadableByteChannel channel = Channels.newChannel(fileContents);

      // Note: Another way to do this is to determine the channel size, and allocate an array of the
      // correct size.
      //       However, this involves additional file system access.

      // Init the output buffer for holding the input stream
      ByteArrayOutputStream baos = threadLocalByteArrayOutputStream.get();
      baos.reset();

      byte[] classData = threadLocalByteArray.get();
      ByteBuffer buffer = ByteBuffer.wrap(classData);

      // Read the input stream to a buffer and write the buffer to the output buffer
      while (true) {
        int bytesInBuffer = channel.read(buffer);
        if (bytesInBuffer < 0) {
          break;
        }
        baos.write(classData, 0, bytesInBuffer);
        buffer.rewind(); // rewind the buffer so that the next read starts at position 0 again
      }

      // Convert the output stream to an byte array and create an input stream
      // based on this array
      return baos.toByteArray();

    } catch (IOException e) {
      // Simply fall through to the finally block.

    } finally {
      if (fileContents != null) {
        try {
          fileContents.close();
        } catch (IOException e) {
          // Not really anything to do at this point.  Fall through to return null.
        }
      }
    }

    return null;
  }
示例#17
0
 /**
  * Downloads from an url and store the image.
  *
  * @param idImage
  * @param url
  */
 public void storeImage(String idImage, String url) {
   try {
     URL website = new URL(url);
     Path pathToFile = Paths.get(getRoot());
     ReadableByteChannel rbc;
     rbc = Channels.newChannel(website.openStream());
     FileOutputStream fos = new FileOutputStream(pathToFile.resolve(idImage).toFile());
     fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
     fos.flush();
     fos.close();
     rbc.close();
   } catch (IOException e) {
     e.printStackTrace();
   }
 }
  public byte[] getfromUrl(String urlName) throws MalformedURLException, IOException {
    int totalRead = 0;
    URL url = new URL(urlName);
    ReadableByteChannel rbc = Channels.newChannel(url.openStream());
    ByteBuffer buf = ByteBuffer.allocateDirect(BUFFER_SIZE);

    while (rbc.read(buf) != -1) {}
    buf.flip();
    totalRead = buf.limit();
    LOG.debug("baixado: " + totalRead + " bytes de [" + urlName + "]");
    byte[] b = new byte[totalRead];
    buf.get(b, 0, totalRead);
    rbc.close();
    return b;
  }
示例#19
0
  public static void copy(InputStream in, OutputStream out) throws IOException {
    ReadableByteChannel source = Channels.newChannel(in);
    WritableByteChannel target = Channels.newChannel(out);

    ByteBuffer buffer = ByteBuffer.allocate(16 * 1024);
    while (source.read(buffer) != -1) {
      buffer.flip(); // Prepare the buffer to be drained
      while (buffer.hasRemaining()) {
        target.write(buffer);
      }
      buffer.clear(); // Empty buffer to get ready for filling
    }

    source.close();
    target.close();
  }
示例#20
0
  /**
   * Copy the readable byte channel to the writable byte channel
   *
   * @param inChannel the readable byte channel
   * @param outChannel the writable byte channel
   * @throws IOException if an IOException occurs.
   */
  public static void copy(ReadableByteChannel inChannel, WritableByteChannel outChannel)
      throws IOException {

    if (inChannel instanceof FileChannel) {
      ((FileChannel) inChannel).transferTo(0, ((FileChannel) inChannel).size(), outChannel);
    } else {

      final ByteBuffer buffer = ByteBuffer.allocateDirect(16 * 1024);
      try {

        while (inChannel.read(buffer) != -1) {
          // prepare the buffer to be drained
          buffer.flip();
          // write to the channel, may block
          outChannel.write(buffer);
          // If partial transfer, shift remainder down
          // If buffer is empty, same as doing clear()
          buffer.compact();
        }
        // EOF will leave buffer in fill state
        buffer.flip();
        // make sure the buffer is fully drained.
        while (buffer.hasRemaining()) {
          outChannel.write(buffer);
        }
      } finally {
        IOUtils.close(inChannel);
        IOUtils.close(outChannel);
      }
    }
  }
    public boolean onData(INonBlockingConnection connection)
        throws IOException, BufferUnderflowException, MaxReadSizeExceededException {

      connection.readStringByDelimiter("\r\n");

      File file = QAUtil.createTestfile_400k();
      RandomAccessFile raf = new RandomAccessFile(file, "r");
      ReadableByteChannel in = raf.getChannel();
      connection.transferFrom(in);

      in.close();
      raf.close();
      file.delete();

      return true;
    }
示例#22
0
 /**
  * Platform-independent channel-to-channel transfer method. Uses regular {@code read} and {@code
  * write} operations to move bytes from the {@code source} channel to the {@code sink} channel.
  * After this call, the {@code throughBuffer} should be checked for remaining bytes; if there are
  * any, they should be written to the {@code sink} channel before proceeding. This method may be
  * used with NIO channels, XNIO channels, or a combination of the two.
  *
  * <p>If either or both of the given channels are blocking channels, then this method may block.
  *
  * @param source the source channel to read bytes from
  * @param count the number of bytes to transfer (must be >= {@code 0L})
  * @param throughBuffer the buffer to transfer through (must not be {@code null})
  * @param sink the sink channel to write bytes to
  * @return the number of bytes actually transferred (possibly 0)
  * @throws IOException if an I/O error occurs during the transfer of bytes
  */
 public static long transfer(
     final ReadableByteChannel source,
     final long count,
     final ByteBuffer throughBuffer,
     final WritableByteChannel sink)
     throws IOException {
   long res;
   long total = 0L;
   throughBuffer.limit(0);
   while (total < count) {
     throughBuffer.compact();
     try {
       if (count - total < (long) throughBuffer.remaining()) {
         throughBuffer.limit((int) (count - total));
       }
       res = source.read(throughBuffer);
       if (res <= 0) {
         return total == 0L ? res : total;
       }
     } finally {
       throughBuffer.flip();
     }
     res = sink.write(throughBuffer);
     if (res == 0) {
       return total;
     }
     total += res;
   }
   return total;
 }
示例#23
0
 /**
  * Clean up any resources. Closes the channel.
  *
  * @throws IOException If errors occur while closing the channel.
  */
 public void close() throws IOException {
   // don't throw NPE on double close
   if (channel == null) return;
   try {
     if (channel.isOpen()) {
       channel.close();
       streamLogger.close();
     }
     NIOUtilities.clean(buffer, useMemoryMappedBuffer);
   } finally {
     if (shxReader != null) shxReader.close();
   }
   shxReader = null;
   channel = null;
   header = null;
 }
示例#24
0
  /**
   * This is a wrapper around {@link ReadableByteChannel#read(ByteBuffer)}. If the amount of data is
   * large, it writes to channel in smaller chunks. This is to avoid jdk from creating many direct
   * buffers as the size of ByteBuffer increases. There should not be any performance degredation.
   *
   * @see ReadableByteChannel#read(ByteBuffer)
   */
  private static int channelRead(ReadableByteChannel channel, ByteBuffer buffer)
      throws IOException {

    return (buffer.remaining() <= NIO_BUFFER_LIMIT)
        ? channel.read(buffer)
        : channelIO(channel, null, buffer);
  }
示例#25
0
  /**
   * Helper for {@link #channelRead(ReadableByteChannel, ByteBuffer)} and {@link
   * #channelWrite(WritableByteChannel, ByteBuffer)}. Only one of readCh or writeCh should be
   * non-null.
   *
   * @see #channelRead(ReadableByteChannel, ByteBuffer)
   * @see #channelWrite(WritableByteChannel, ByteBuffer)
   */
  private static int channelIO(
      ReadableByteChannel readCh, WritableByteChannel writeCh, ByteBuffer buf) throws IOException {

    int originalLimit = buf.limit();
    int initialRemaining = buf.remaining();
    int ret = 0;

    while (buf.remaining() > 0) {
      try {
        int ioSize = Math.min(buf.remaining(), NIO_BUFFER_LIMIT);
        buf.limit(buf.position() + ioSize);

        ret = (readCh == null) ? writeCh.write(buf) : readCh.read(buf);

        if (ret < ioSize) {
          break;
        }

      } finally {
        buf.limit(originalLimit);
      }
    }

    int nBytes = initialRemaining - buf.remaining();
    return (nBytes > 0) ? nBytes : ret;
  }
示例#26
0
  private void fetch(String res) throws IOException {
    URL url = new URL(Configuration.composeres() + res);
    HttpURLConnection con = (HttpURLConnection) url.openConnection();
    con.setRequestMethod("GET");
    con.setDoInput(true);
    con.setConnectTimeout(5000);
    con.setRequestProperty(
        "User-Agent",
        "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11");

    ReadableByteChannel rbc = Channels.newChannel(con.getInputStream());
    FileOutputStream fos = new FileOutputStream(Configuration.STORAGE_DIR + File.separator + res);
    fos.getChannel().transferFrom(rbc, 0, 1 << 24);
    fos.close();
    rbc.close();
  }
  private int doRead(ReadableByteChannel channel, ByteBuffer byteBuffer) throws IOException {
    int readBytes = 0;
    int lastReadBytes = -1;
    try {
      do {
        lastReadBytes = channel.read(byteBuffer);
        if (lastReadBytes > 0) {
          readBytes += lastReadBytes;
        } else if (lastReadBytes == -1 && readBytes == 0) {
          readBytes = -1;
        }

      } while (lastReadBytes > 0 && byteBuffer.hasRemaining());
    } catch (IOException ex) {
      lastReadBytes = -1;
      throw ex;
    } finally {
      if (lastReadBytes == -1 || readBytes == -1) {
        SelectionKeyHandler skh = selectorHandler.getSelectionKeyHandler();
        if (skh instanceof BaseSelectionKeyHandler) {
          ((BaseSelectionKeyHandler) skh)
              .notifyRemotlyClose(
                  ((SelectableChannel) channel).keyFor(selectorHandler.getSelector()));
        }
      }
    }

    return readBytes;
  }
示例#28
0
 public void write(
     String path, ReadableByteChannel data, ByteBuffer temporaryBuffer, boolean hasData)
     throws IOException {
   try {
     File file = new File(basePath, path);
     RandomAccessFile randomAccessFile = null;
     try {
       file.getParentFile().mkdirs();
       randomAccessFile = new RandomAccessFile(file, "rw");
       if (hasData) {
         FileChannel channel = randomAccessFile.getChannel();
         while (data.read(temporaryBuffer) >= 0) {
           temporaryBuffer.flip();
           channel.write(temporaryBuffer);
           temporaryBuffer.clear();
         }
       }
     } finally {
       if (randomAccessFile != null) {
         randomAccessFile.close();
       }
     }
   } catch (Throwable t) {
     t.printStackTrace();
     throw new IOException(t);
   }
 }
 @Test(expected = IOException.class)
 public void testTruncatedData() throws IOException {
   // with BZ2File(self.filename) as f:
   //    self.assertRaises(EOFError, f.read)
   System.out.println("Attempt to read the whole thing in, should throw ...");
   ByteBuffer buffer = ByteBuffer.allocate(8192);
   bz2Channel.read(buffer);
 }
示例#30
0
 public static int readToBuffer(ReadableByteChannel channel, ByteBuffer buffer)
     throws NetIOException {
   try {
     return channel.read(buffer);
   } catch (IOException e) {
     return -1;
   }
 }