Beispiel #1
0
 /** Creates an IsmSinkWriter for the given channel. */
 private IsmSinkWriter(WritableByteChannel channel) {
   checkNotNull(channel);
   out = new CountingOutputStream(Channels.newOutputStream(channel));
   indexOut = new RandomAccessData();
   lastKeyBytes = new RandomAccessData();
   currentKeyBytes = new RandomAccessData();
   lastIndexKeyBytes = new RandomAccessData();
   bloomFilterBuilder = ScalableBloomFilter.builder();
 }
Beispiel #2
0
 /**
  * Constructor.
  *
  * @param socketChannel the SocketChannel underlying this SocketCommChannel
  * @param location the location for this channel
  * @param protocol the CommProtocol to use to send and receive messages
  * @throws java.io.IOException
  * @see CommProtocol
  * @see SocketChannel
  */
 public SocketCommChannel(SocketChannel socketChannel, URI location, CommProtocol protocol)
     throws IOException {
   super(location, protocol);
   this.socketChannel = socketChannel;
   socketChannel.socket().setSoLinger(true, SO_LINGER);
   this.istream =
       new PreBufferedInputStream(new BufferedInputStream(Channels.newInputStream(socketChannel)));
   this.ostream = new BufferedOutputStream(Channels.newOutputStream(socketChannel));
   setToBeClosed(false); // Socket connections are kept open by default
 }
Beispiel #3
0
 /** @deprecated use read( ByteBuffer ) / write( ByteBuffer ); */
 @Deprecated
 public GnutellaOutputStream getOutputStream() throws IOException {
   if (outputStream == null) {
     if (socket == null) {
       throw new ConnectionClosedException("Connection already closed");
     }
     initBandwidthByteChannel();
     OutputStream outStream = Channels.newOutputStream(bandwidthByteChannel);
     outputStream = new GnutellaOutputStream(outStream);
   }
   return outputStream;
 }
Beispiel #4
0
 public OutputStream newFileOutputStream(Path pathRelativeToProjectRoot, FileAttribute<?>... attrs)
     throws IOException {
   return new BufferedOutputStream(
       Channels.newOutputStream(
           Files.newByteChannel(
               getPathForRelativePath(pathRelativeToProjectRoot),
               ImmutableSet.<OpenOption>of(
                   StandardOpenOption.CREATE,
                   StandardOpenOption.TRUNCATE_EXISTING,
                   StandardOpenOption.WRITE),
               attrs)));
 }
  public AnalysisStatusUpdater(final String host, final int port) throws IOException {
    EvaluatorLoggingHandler.logger.info("Listening for server status on port " + port);
    this.socketConnection = SocketChannel.open(new InetSocketAddress(host, port));

    this.socketConnection.socket().setSoTimeout(250);

    this.out = new PrintWriter(Channels.newOutputStream(this.socketConnection));
    this.in =
        new BufferedReader(new InputStreamReader(Channels.newInputStream(this.socketConnection)));

    this.completedState = QueryConstants.failedComplete;
    this.finished = false;
  }
  @FixFor("MODE-1358")
  @Test
  public void shouldCopyFilesUsingStreams() throws Exception {
    // Copy a large file into a temporary file ...
    File tempFile = File.createTempFile("copytest", "pdf");
    RandomAccessFile destinationRaf = null;
    RandomAccessFile originalRaf = null;
    try {
      URL sourceUrl = getClass().getResource("/docs/postgresql-8.4.1-US.pdf");
      assertThat(sourceUrl, is(notNullValue()));
      File sourceFile = new File(sourceUrl.toURI());
      assertThat(sourceFile.exists(), is(true));
      assertThat(sourceFile.canRead(), is(true));
      assertThat(sourceFile.isFile(), is(true));

      boolean useBufferedStream = true;
      final int bufferSize = AbstractBinaryStore.bestBufferSize(sourceFile.length());

      destinationRaf = new RandomAccessFile(tempFile, "rw");
      originalRaf = new RandomAccessFile(sourceFile, "r");

      FileChannel destinationChannel = destinationRaf.getChannel();
      OutputStream output = Channels.newOutputStream(destinationChannel);
      if (useBufferedStream) output = new BufferedOutputStream(output, bufferSize);

      // Create an input stream to the original file ...
      FileChannel originalChannel = originalRaf.getChannel();
      InputStream input = Channels.newInputStream(originalChannel);
      if (useBufferedStream) input = new BufferedInputStream(input, bufferSize);

      // Copy the content ...
      Stopwatch sw = new Stopwatch();
      sw.start();
      IoUtil.write(input, output, bufferSize);
      sw.stop();
      System.out.println(
          "Time to copy \""
              + sourceFile.getName()
              + "\" ("
              + sourceFile.length()
              + " bytes): "
              + sw.getTotalDuration());
    } finally {
      tempFile.delete();
      if (destinationRaf != null) destinationRaf.close();
      if (originalRaf != null) originalRaf.close();
    }
  }
  /**
   * Re-open the output stream of the tlog and position the file pointer at the end of the file. It
   * assumes that the tlog is non-empty and that the tlog's header has been already read.
   */
  synchronized void reopenOutputStream() {
    try {
      if (debug) {
        log.debug("Re-opening tlog's output stream: " + this);
      }

      raf = new RandomAccessFile(this.tlogFile, "rw");
      channel = raf.getChannel();
      long start = raf.length();
      raf.seek(start);
      os = Channels.newOutputStream(channel);
      fos = new FastOutputStream(os, new byte[65536], 0);
      fos.setWritten(start); // reflect that we aren't starting at the beginning
    } catch (IOException e) {
      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
    }
  }
  @Override
  public void download(String id, WritableByteChannel target) throws IOException {
    Path tmpFile = Files.createTempFile(tmpDir, id, ".unzipped");

    try (FileChannel tmpChannel = FileChannel.open(tmpFile, FileUtils.TMP_FILE_OPEN_OPTIONS)) {
      underlyingStore.download(id, tmpChannel);

      // Reset channel for reading
      tmpChannel.position(0);

      GZIPInputStream in = new GZIPInputStream(Channels.newInputStream(tmpChannel));
      OutputStream out = Channels.newOutputStream(target);

      IOUtils.copy(in, out);

      logger.debug("Data '{}' successfully unzipped", id);
    }
  }
 /*     */ public OutputStream newOutputStream(Path paramPath, OpenOption[] paramArrayOfOpenOption)
     /*     */ throws IOException
       /*     */ {
   /* 417 */ int i = paramArrayOfOpenOption.length;
   /* 418 */ HashSet localHashSet = new HashSet(i + 3);
   /* 419 */ if (i == 0) {
     /* 420 */ localHashSet.add(StandardOpenOption.CREATE);
     /* 421 */ localHashSet.add(StandardOpenOption.TRUNCATE_EXISTING);
     /*     */ } else {
     /* 423 */ for (OpenOption localOpenOption : paramArrayOfOpenOption) {
       /* 424 */ if (localOpenOption == StandardOpenOption.READ)
         /* 425 */ throw new IllegalArgumentException("READ not allowed");
       /* 426 */ localHashSet.add(localOpenOption);
       /*     */ }
     /*     */ }
   /* 429 */ localHashSet.add(StandardOpenOption.WRITE);
   /* 430 */ return Channels.newOutputStream(
       newByteChannel(paramPath, localHashSet, new FileAttribute[0]));
   /*     */ }
  @Override
  public void upload(String id, ReadableByteChannel src, long length) throws IOException {
    Path tmpFile = Files.createTempFile(tmpDir, id, ".zipped");

    try (FileChannel tmpChannel = FileChannel.open(tmpFile, FileUtils.TMP_FILE_OPEN_OPTIONS)) {
      InputStream in = Channels.newInputStream(src);
      GZIPOutputStream out = new GZIPOutputStream(Channels.newOutputStream(tmpChannel));

      IOUtils.copy(in, out);

      out.finish();

      logger.debug("Data '{}' successfully zipped", id);

      // Reset channel for reading
      tmpChannel.position(0);

      underlyingStore.upload(id, tmpChannel, tmpChannel.size());
    }
  }
  @SuppressWarnings("resource")
  protected void openFile() throws IOException {
    synchronized (locked) {
      while (locked.containsKey(filename) && locked.get(filename)) {
        try {
          locked.wait();
        } catch (InterruptedException e) {
        }
      }
      locked.put(filename, true);

      File file = new File(this.filename);
      if (!file.exists()) {
        locked.put(filename, false);
        locked.notifyAll();
        throw new IllegalStateException(
            "Warning: File doesn't exist (anymore):'" + this.filename + "'");
      }

      channel = new RandomAccessFile(file, "rw").getChannel();
      try {
        // TODO: add support for shared locks, allowing parallel reading
        // operations.
        lock = channel.lock();

      } catch (Exception e) {
        channel.close();
        channel = null;
        lock = null;
        locked.put(filename, false);
        locked.notifyAll();
        throw new IllegalStateException("error, couldn't obtain file lock on:" + filename, e);
      }
      fis = new BufferedInputStream(Channels.newInputStream(channel));
      fos = new BufferedOutputStream(Channels.newOutputStream(channel));
    }
  }
 public DataOutputStream openDataOutputStream() throws IOException {
   if (this.dos == null) this.dos = new DataOutputStream(Channels.newOutputStream(this.channel));
   return this.dos;
 }
Beispiel #13
0
  public static void main(String[] args) throws IOException {
    ChannelRouter channelRouter =
        new ChannelRouter(
            new ExceptionListener<IOException>() {
              @Override
              public void exceptionThrown(IOException exception) {
                exception.printStackTrace();
              }
            });

    int port = Integer.parseInt(args[1]);
    SocketChannel socketChannel;
    if (args[0].equals("--server")) {
      System.out.println("Server mode started; listening on port " + port);
      ServerSocketChannel serverSocketChannel = ServerSocketChannel.open();
      serverSocketChannel.socket().bind(new InetSocketAddress(port));
      socketChannel = serverSocketChannel.accept();
      System.out.println("Client connected");
      serverSocketChannel.close();
    } else {
      String host = args[0];
      System.out.println("Client mode started; connecting to " + host + " on port " + port);
      socketChannel = SocketChannel.open(new InetSocketAddress(host, port));
      System.out.println("Connected to server");
    }
    socketChannel.configureBlocking(false);

    Pipe localToRemote = Pipe.open();
    localToRemote.source().configureBlocking(false);
    channelRouter.addRoute(localToRemote.source(), socketChannel);

    final Pipe remoteToLocal = Pipe.open();
    remoteToLocal.sink().configureBlocking(false);
    channelRouter.addRoute(socketChannel, remoteToLocal.sink());

    Thread channelRouterThread = new Thread(channelRouter, "ChannelRouter");
    channelRouterThread.start();

    Thread remoteReaderThread =
        new Thread("RemoteReader") {
          @Override
          public void run() {
            try {
              // BufferedReader.readLine() blocks until an end-of-line is
              // written, so make sure they get written by the main thread
              InputStream temporaryInputStream = Channels.newInputStream(remoteToLocal.source());
              BufferedReader in = new BufferedReader(new InputStreamReader(temporaryInputStream));
              String line = in.readLine();
              while (null != line) {
                System.out.println(line);
                line = in.readLine();
              }
            } catch (ClosedByInterruptException e) {
              // The main thread wants us to close, so do nothing
            } catch (IOException e) {
              e.printStackTrace();
            }
          }
        };
    remoteReaderThread.start();

    BufferedReader in = new BufferedReader(new InputStreamReader(System.in));

    Writer temporaryWriter = new OutputStreamWriter(Channels.newOutputStream(localToRemote.sink()));
    temporaryWriter = new BufferedWriter(temporaryWriter);
    PrintWriter out = new PrintWriter(temporaryWriter, true);

    System.out.println(
        "Press end-of-file to terminate (Ctrl+Z on Windows, Ctrl+D on other platforms)");
    String line = in.readLine();
    while (null != line) {
      // The end-of-line is required for BufferedReader.readLine() to
      // return and this will automatically flush due to the auto-flush
      // parameter on construction
      out.println(line);
      line = in.readLine();
    }

    remoteReaderThread.interrupt();
    channelRouterThread.interrupt();
    socketChannel.close();
  }
Beispiel #14
0
 /**
  * A convenience method for the following code:
  *
  * <pre>
  * {@link OutputStream} tempOutputStream = {@link Channels#newOutputStream(WritableByteChannel) Channels.newOutputStream(output)};
  * tempOutputStream = new {@link BufferedOutputStream#BufferedOutputStream(OutputStream) java.io.BufferedOutputStream(tempOutputStream)};
  * return new {@link ObjectOutputStream#ObjectOutputStream(OutputStream) java.io.ObjectOutputStream(tempOutputStream)};
  * </pre>
  *
  * @throws IOException pass-through for any {@link IOException} that occurs in the above code
  */
 public static ObjectOutputStream wrapChannelInObjectOutputStream(WritableByteChannel output)
     throws IOException {
   OutputStream tempOutputStream = Channels.newOutputStream(output);
   tempOutputStream = new BufferedOutputStream(tempOutputStream);
   return new ObjectOutputStream(tempOutputStream);
 }
Beispiel #15
0
  /**
   * This will copy the template directory, renaming files named {@code foo.fixture} to {@code foo}
   * in the process. Files whose names end in {@code .expected} will not be copied.
   */
  @SuppressWarnings("PMD.EmptyCatchBlock")
  public ProjectWorkspace setUp() throws IOException {

    MoreFiles.copyRecursively(templatePath, destPath, BUILD_FILE_RENAME);

    // Stamp the buck-out directory if it exists and isn't stamped already
    try (OutputStream outputStream =
        new BufferedOutputStream(
            Channels.newOutputStream(
                Files.newByteChannel(
                    destPath.resolve(BuckConstant.getCurrentVersionFile()),
                    ImmutableSet.<OpenOption>of(
                        StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE))))) {
      outputStream.write(BuckVersion.getVersion().getBytes(Charsets.UTF_8));
    } catch (FileAlreadyExistsException | NoSuchFileException e) {
      // If the current version file already exists we don't need to create it
      // If buck-out doesn't exist we don't need to stamp it
    }

    if (Platform.detect() == Platform.WINDOWS) {
      // Hack for symlinks on Windows.
      SimpleFileVisitor<Path> copyDirVisitor =
          new SimpleFileVisitor<Path>() {
            @Override
            public FileVisitResult visitFile(Path path, BasicFileAttributes attrs)
                throws IOException {
              // On Windows, symbolic links from git repository are checked out as normal files
              // containing a one-line path. In order to distinguish them, paths are read and
              // pointed
              // files are trued to locate. Once the pointed file is found, it will be copied to
              // target.
              // On NTFS length of path must be greater than 0 and less than 4096.
              if (attrs.size() > 0 && attrs.size() <= 4096) {
                String linkTo = new String(Files.readAllBytes(path), UTF_8);
                Path linkToFile;
                try {
                  linkToFile = templatePath.resolve(linkTo);
                } catch (InvalidPathException e) {
                  // Let's assume we were reading a normal text file, and not something meant to be
                  // a
                  // link.
                  return FileVisitResult.CONTINUE;
                }
                if (Files.isRegularFile(linkToFile)) {
                  Files.copy(linkToFile, path, StandardCopyOption.REPLACE_EXISTING);
                } else if (Files.isDirectory(linkToFile)) {
                  Files.delete(path);
                  MoreFiles.copyRecursively(linkToFile, path);
                }
              }
              return FileVisitResult.CONTINUE;
            }
          };
      Files.walkFileTree(destPath, copyDirVisitor);
    }

    // Disable the directory cache by default.  Tests that want to enable it can call
    // `enableDirCache` on this object.  Only do this if a .buckconfig.local file does not already
    // exist, however (we assume the test knows what it is doing at that point).
    if (!Files.exists(getPath(".buckconfig.local"))) {
      writeContentsToPath("[cache]\n  mode =", ".buckconfig.local");
    }

    isSetUp = true;
    return this;
  }
Beispiel #16
0
  TransactionLog(File tlogFile, Collection<String> globalStrings, boolean openExisting) {
    boolean success = false;
    try {
      if (debug) {
        log.debug(
            "New TransactionLog file="
                + tlogFile
                + ", exists="
                + tlogFile.exists()
                + ", size="
                + tlogFile.length()
                + ", openExisting="
                + openExisting);
      }

      this.tlogFile = tlogFile;
      raf = new RandomAccessFile(this.tlogFile, "rw");
      long start = raf.length();
      channel = raf.getChannel();
      os = Channels.newOutputStream(channel);
      fos = new FastOutputStream(os, new byte[65536], 0);
      // fos = FastOutputStream.wrap(os);

      if (openExisting) {
        if (start > 0) {
          readHeader(null);
          raf.seek(start);
          assert channel.position() == start;
          fos.setWritten(start); // reflect that we aren't starting at the beginning
          assert fos.size() == channel.size();
        } else {
          addGlobalStrings(globalStrings);
        }
      } else {
        if (start > 0) {
          log.warn("New transaction log already exists:" + tlogFile + " size=" + raf.length());
          return;
        }

        if (start > 0) {
          raf.setLength(0);
        }
        addGlobalStrings(globalStrings);
      }

      success = true;

      assert ObjectReleaseTracker.track(this);

    } catch (IOException e) {
      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
    } finally {
      if (!success && raf != null) {
        try {
          raf.close();
        } catch (Exception e) {
          log.error("Error closing tlog file (after error opening)", e);
        }
      }
    }
  }
Beispiel #17
0
 @SuppressWarnings("deprecation") // uses internal test functionality.
 @Override
 protected void prepareWrite(WritableByteChannel channel) throws Exception {
   dataFileWriter = new DataFileWriter<>(coder.createDatumWriter());
   dataFileWriter.create(coder.getSchema(), Channels.newOutputStream(channel));
 }
Beispiel #18
0
  /**
   * This will copy the template directory, renaming files named {@code foo.fixture} to {@code foo}
   * in the process. Files whose names end in {@code .expected} will not be copied.
   */
  @SuppressWarnings("PMD.EmptyCatchBlock")
  public ProjectWorkspace setUp() throws IOException {

    MoreFiles.copyRecursively(templatePath, destPath, BUILD_FILE_RENAME);

    // Stamp the buck-out directory if it exists and isn't stamped already
    try (OutputStream outputStream =
        new BufferedOutputStream(
            Channels.newOutputStream(
                Files.newByteChannel(
                    destPath.resolve(BuckConstant.CURRENT_VERSION_FILE),
                    ImmutableSet.<OpenOption>of(
                        StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE))))) {
      outputStream.write(BuckVersion.getVersion().getBytes(Charsets.UTF_8));
    } catch (FileAlreadyExistsException | NoSuchFileException e) {
      // If the current version file already exists we don't need to create it
      // If buck-out doesn't exist we don't need to stamp it
    }

    // If there's a local.properties in the host project but not in the destination, make a copy.
    Path localProperties = FileSystems.getDefault().getPath("local.properties");
    Path destLocalProperties = destPath.resolve(localProperties.getFileName());

    if (localProperties.toFile().exists() && !destLocalProperties.toFile().exists()) {
      Files.copy(localProperties, destLocalProperties);
    }

    if (Platform.detect() == Platform.WINDOWS) {
      // Hack for symlinks on Windows.
      SimpleFileVisitor<Path> copyDirVisitor =
          new SimpleFileVisitor<Path>() {
            @Override
            public FileVisitResult visitFile(Path path, BasicFileAttributes attrs)
                throws IOException {
              // On Windows, symbolic links from git repository are checked out as normal files
              // containing a one-line path. In order to distinguish them, paths are read and
              // pointed
              // files are trued to locate. Once the pointed file is found, it will be copied to
              // target.
              // On NTFS length of path must be greater than 0 and less than 4096.
              if (attrs.size() > 0 && attrs.size() <= 4096) {
                String linkTo = new String(Files.readAllBytes(path), UTF_8);
                Path linkToFile = null;
                try {
                  linkToFile = templatePath.resolve(linkTo);
                } catch (InvalidPathException e) {
                  // Let's assume we were reading a normal text file, and not something meant to be
                  // a
                  // link.
                  return FileVisitResult.CONTINUE;
                }
                if (Files.isRegularFile(linkToFile)) {
                  Files.copy(linkToFile, path, StandardCopyOption.REPLACE_EXISTING);
                } else if (Files.isDirectory(linkToFile)) {
                  Files.delete(path);
                  MoreFiles.copyRecursively(linkToFile, path);
                }
              }
              return FileVisitResult.CONTINUE;
            }
          };
      Files.walkFileTree(destPath, copyDirVisitor);
    }
    isSetUp = true;
    return this;
  }
Beispiel #19
0
 /**
  * Returns an output stream based on a given writable byte channel.
  *
  * @param writableChannel The writable byte channel.
  * @return An output stream based on a given writable byte channel.
  */
 public static OutputStream getStream(WritableByteChannel writableChannel) {
   return isBlocking(writableChannel)
       ? Channels.newOutputStream(writableChannel)
       : new NbChannelOutputStream(writableChannel);
 }
 @Override
 public OutputStream getOutputStream() throws IOException {
   return Channels.newOutputStream(inner);
 }