/** Creates an IsmSinkWriter for the given channel. */
private IsmSinkWriter(WritableByteChannel channel) {
checkNotNull(channel);
out = new CountingOutputStream(Channels.newOutputStream(channel));
indexOut = new RandomAccessData();
lastKeyBytes = new RandomAccessData();
currentKeyBytes = new RandomAccessData();
lastIndexKeyBytes = new RandomAccessData();
bloomFilterBuilder = ScalableBloomFilter.builder();
}
/**
* Constructor.
*
* @param socketChannel the SocketChannel underlying this SocketCommChannel
* @param location the location for this channel
* @param protocol the CommProtocol to use to send and receive messages
* @throws java.io.IOException
* @see CommProtocol
* @see SocketChannel
*/
public SocketCommChannel(SocketChannel socketChannel, URI location, CommProtocol protocol)
throws IOException {
super(location, protocol);
this.socketChannel = socketChannel;
socketChannel.socket().setSoLinger(true, SO_LINGER);
this.istream =
new PreBufferedInputStream(new BufferedInputStream(Channels.newInputStream(socketChannel)));
this.ostream = new BufferedOutputStream(Channels.newOutputStream(socketChannel));
setToBeClosed(false); // Socket connections are kept open by default
}
/** @deprecated use read( ByteBuffer ) / write( ByteBuffer ); */
@Deprecated
public GnutellaOutputStream getOutputStream() throws IOException {
if (outputStream == null) {
if (socket == null) {
throw new ConnectionClosedException("Connection already closed");
}
initBandwidthByteChannel();
OutputStream outStream = Channels.newOutputStream(bandwidthByteChannel);
outputStream = new GnutellaOutputStream(outStream);
}
return outputStream;
}
public OutputStream newFileOutputStream(Path pathRelativeToProjectRoot, FileAttribute<?>... attrs)
throws IOException {
return new BufferedOutputStream(
Channels.newOutputStream(
Files.newByteChannel(
getPathForRelativePath(pathRelativeToProjectRoot),
ImmutableSet.<OpenOption>of(
StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING,
StandardOpenOption.WRITE),
attrs)));
}
public AnalysisStatusUpdater(final String host, final int port) throws IOException {
EvaluatorLoggingHandler.logger.info("Listening for server status on port " + port);
this.socketConnection = SocketChannel.open(new InetSocketAddress(host, port));
this.socketConnection.socket().setSoTimeout(250);
this.out = new PrintWriter(Channels.newOutputStream(this.socketConnection));
this.in =
new BufferedReader(new InputStreamReader(Channels.newInputStream(this.socketConnection)));
this.completedState = QueryConstants.failedComplete;
this.finished = false;
}
@FixFor("MODE-1358")
@Test
public void shouldCopyFilesUsingStreams() throws Exception {
// Copy a large file into a temporary file ...
File tempFile = File.createTempFile("copytest", "pdf");
RandomAccessFile destinationRaf = null;
RandomAccessFile originalRaf = null;
try {
URL sourceUrl = getClass().getResource("/docs/postgresql-8.4.1-US.pdf");
assertThat(sourceUrl, is(notNullValue()));
File sourceFile = new File(sourceUrl.toURI());
assertThat(sourceFile.exists(), is(true));
assertThat(sourceFile.canRead(), is(true));
assertThat(sourceFile.isFile(), is(true));
boolean useBufferedStream = true;
final int bufferSize = AbstractBinaryStore.bestBufferSize(sourceFile.length());
destinationRaf = new RandomAccessFile(tempFile, "rw");
originalRaf = new RandomAccessFile(sourceFile, "r");
FileChannel destinationChannel = destinationRaf.getChannel();
OutputStream output = Channels.newOutputStream(destinationChannel);
if (useBufferedStream) output = new BufferedOutputStream(output, bufferSize);
// Create an input stream to the original file ...
FileChannel originalChannel = originalRaf.getChannel();
InputStream input = Channels.newInputStream(originalChannel);
if (useBufferedStream) input = new BufferedInputStream(input, bufferSize);
// Copy the content ...
Stopwatch sw = new Stopwatch();
sw.start();
IoUtil.write(input, output, bufferSize);
sw.stop();
System.out.println(
"Time to copy \""
+ sourceFile.getName()
+ "\" ("
+ sourceFile.length()
+ " bytes): "
+ sw.getTotalDuration());
} finally {
tempFile.delete();
if (destinationRaf != null) destinationRaf.close();
if (originalRaf != null) originalRaf.close();
}
}
/**
* Re-open the output stream of the tlog and position the file pointer at the end of the file. It
* assumes that the tlog is non-empty and that the tlog's header has been already read.
*/
synchronized void reopenOutputStream() {
try {
if (debug) {
log.debug("Re-opening tlog's output stream: " + this);
}
raf = new RandomAccessFile(this.tlogFile, "rw");
channel = raf.getChannel();
long start = raf.length();
raf.seek(start);
os = Channels.newOutputStream(channel);
fos = new FastOutputStream(os, new byte[65536], 0);
fos.setWritten(start); // reflect that we aren't starting at the beginning
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
}
@Override
public void download(String id, WritableByteChannel target) throws IOException {
Path tmpFile = Files.createTempFile(tmpDir, id, ".unzipped");
try (FileChannel tmpChannel = FileChannel.open(tmpFile, FileUtils.TMP_FILE_OPEN_OPTIONS)) {
underlyingStore.download(id, tmpChannel);
// Reset channel for reading
tmpChannel.position(0);
GZIPInputStream in = new GZIPInputStream(Channels.newInputStream(tmpChannel));
OutputStream out = Channels.newOutputStream(target);
IOUtils.copy(in, out);
logger.debug("Data '{}' successfully unzipped", id);
}
}
/* */ public OutputStream newOutputStream(Path paramPath, OpenOption[] paramArrayOfOpenOption)
/* */ throws IOException
/* */ {
/* 417 */ int i = paramArrayOfOpenOption.length;
/* 418 */ HashSet localHashSet = new HashSet(i + 3);
/* 419 */ if (i == 0) {
/* 420 */ localHashSet.add(StandardOpenOption.CREATE);
/* 421 */ localHashSet.add(StandardOpenOption.TRUNCATE_EXISTING);
/* */ } else {
/* 423 */ for (OpenOption localOpenOption : paramArrayOfOpenOption) {
/* 424 */ if (localOpenOption == StandardOpenOption.READ)
/* 425 */ throw new IllegalArgumentException("READ not allowed");
/* 426 */ localHashSet.add(localOpenOption);
/* */ }
/* */ }
/* 429 */ localHashSet.add(StandardOpenOption.WRITE);
/* 430 */ return Channels.newOutputStream(
newByteChannel(paramPath, localHashSet, new FileAttribute[0]));
/* */ }
@Override
public void upload(String id, ReadableByteChannel src, long length) throws IOException {
Path tmpFile = Files.createTempFile(tmpDir, id, ".zipped");
try (FileChannel tmpChannel = FileChannel.open(tmpFile, FileUtils.TMP_FILE_OPEN_OPTIONS)) {
InputStream in = Channels.newInputStream(src);
GZIPOutputStream out = new GZIPOutputStream(Channels.newOutputStream(tmpChannel));
IOUtils.copy(in, out);
out.finish();
logger.debug("Data '{}' successfully zipped", id);
// Reset channel for reading
tmpChannel.position(0);
underlyingStore.upload(id, tmpChannel, tmpChannel.size());
}
}
@SuppressWarnings("resource")
protected void openFile() throws IOException {
synchronized (locked) {
while (locked.containsKey(filename) && locked.get(filename)) {
try {
locked.wait();
} catch (InterruptedException e) {
}
}
locked.put(filename, true);
File file = new File(this.filename);
if (!file.exists()) {
locked.put(filename, false);
locked.notifyAll();
throw new IllegalStateException(
"Warning: File doesn't exist (anymore):'" + this.filename + "'");
}
channel = new RandomAccessFile(file, "rw").getChannel();
try {
// TODO: add support for shared locks, allowing parallel reading
// operations.
lock = channel.lock();
} catch (Exception e) {
channel.close();
channel = null;
lock = null;
locked.put(filename, false);
locked.notifyAll();
throw new IllegalStateException("error, couldn't obtain file lock on:" + filename, e);
}
fis = new BufferedInputStream(Channels.newInputStream(channel));
fos = new BufferedOutputStream(Channels.newOutputStream(channel));
}
}
public DataOutputStream openDataOutputStream() throws IOException {
if (this.dos == null) this.dos = new DataOutputStream(Channels.newOutputStream(this.channel));
return this.dos;
}
public static void main(String[] args) throws IOException {
ChannelRouter channelRouter =
new ChannelRouter(
new ExceptionListener<IOException>() {
@Override
public void exceptionThrown(IOException exception) {
exception.printStackTrace();
}
});
int port = Integer.parseInt(args[1]);
SocketChannel socketChannel;
if (args[0].equals("--server")) {
System.out.println("Server mode started; listening on port " + port);
ServerSocketChannel serverSocketChannel = ServerSocketChannel.open();
serverSocketChannel.socket().bind(new InetSocketAddress(port));
socketChannel = serverSocketChannel.accept();
System.out.println("Client connected");
serverSocketChannel.close();
} else {
String host = args[0];
System.out.println("Client mode started; connecting to " + host + " on port " + port);
socketChannel = SocketChannel.open(new InetSocketAddress(host, port));
System.out.println("Connected to server");
}
socketChannel.configureBlocking(false);
Pipe localToRemote = Pipe.open();
localToRemote.source().configureBlocking(false);
channelRouter.addRoute(localToRemote.source(), socketChannel);
final Pipe remoteToLocal = Pipe.open();
remoteToLocal.sink().configureBlocking(false);
channelRouter.addRoute(socketChannel, remoteToLocal.sink());
Thread channelRouterThread = new Thread(channelRouter, "ChannelRouter");
channelRouterThread.start();
Thread remoteReaderThread =
new Thread("RemoteReader") {
@Override
public void run() {
try {
// BufferedReader.readLine() blocks until an end-of-line is
// written, so make sure they get written by the main thread
InputStream temporaryInputStream = Channels.newInputStream(remoteToLocal.source());
BufferedReader in = new BufferedReader(new InputStreamReader(temporaryInputStream));
String line = in.readLine();
while (null != line) {
System.out.println(line);
line = in.readLine();
}
} catch (ClosedByInterruptException e) {
// The main thread wants us to close, so do nothing
} catch (IOException e) {
e.printStackTrace();
}
}
};
remoteReaderThread.start();
BufferedReader in = new BufferedReader(new InputStreamReader(System.in));
Writer temporaryWriter = new OutputStreamWriter(Channels.newOutputStream(localToRemote.sink()));
temporaryWriter = new BufferedWriter(temporaryWriter);
PrintWriter out = new PrintWriter(temporaryWriter, true);
System.out.println(
"Press end-of-file to terminate (Ctrl+Z on Windows, Ctrl+D on other platforms)");
String line = in.readLine();
while (null != line) {
// The end-of-line is required for BufferedReader.readLine() to
// return and this will automatically flush due to the auto-flush
// parameter on construction
out.println(line);
line = in.readLine();
}
remoteReaderThread.interrupt();
channelRouterThread.interrupt();
socketChannel.close();
}
/**
* A convenience method for the following code:
*
* <pre>
* {@link OutputStream} tempOutputStream = {@link Channels#newOutputStream(WritableByteChannel) Channels.newOutputStream(output)};
* tempOutputStream = new {@link BufferedOutputStream#BufferedOutputStream(OutputStream) java.io.BufferedOutputStream(tempOutputStream)};
* return new {@link ObjectOutputStream#ObjectOutputStream(OutputStream) java.io.ObjectOutputStream(tempOutputStream)};
* </pre>
*
* @throws IOException pass-through for any {@link IOException} that occurs in the above code
*/
public static ObjectOutputStream wrapChannelInObjectOutputStream(WritableByteChannel output)
throws IOException {
OutputStream tempOutputStream = Channels.newOutputStream(output);
tempOutputStream = new BufferedOutputStream(tempOutputStream);
return new ObjectOutputStream(tempOutputStream);
}
/**
* This will copy the template directory, renaming files named {@code foo.fixture} to {@code foo}
* in the process. Files whose names end in {@code .expected} will not be copied.
*/
@SuppressWarnings("PMD.EmptyCatchBlock")
public ProjectWorkspace setUp() throws IOException {
MoreFiles.copyRecursively(templatePath, destPath, BUILD_FILE_RENAME);
// Stamp the buck-out directory if it exists and isn't stamped already
try (OutputStream outputStream =
new BufferedOutputStream(
Channels.newOutputStream(
Files.newByteChannel(
destPath.resolve(BuckConstant.getCurrentVersionFile()),
ImmutableSet.<OpenOption>of(
StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE))))) {
outputStream.write(BuckVersion.getVersion().getBytes(Charsets.UTF_8));
} catch (FileAlreadyExistsException | NoSuchFileException e) {
// If the current version file already exists we don't need to create it
// If buck-out doesn't exist we don't need to stamp it
}
if (Platform.detect() == Platform.WINDOWS) {
// Hack for symlinks on Windows.
SimpleFileVisitor<Path> copyDirVisitor =
new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path path, BasicFileAttributes attrs)
throws IOException {
// On Windows, symbolic links from git repository are checked out as normal files
// containing a one-line path. In order to distinguish them, paths are read and
// pointed
// files are trued to locate. Once the pointed file is found, it will be copied to
// target.
// On NTFS length of path must be greater than 0 and less than 4096.
if (attrs.size() > 0 && attrs.size() <= 4096) {
String linkTo = new String(Files.readAllBytes(path), UTF_8);
Path linkToFile;
try {
linkToFile = templatePath.resolve(linkTo);
} catch (InvalidPathException e) {
// Let's assume we were reading a normal text file, and not something meant to be
// a
// link.
return FileVisitResult.CONTINUE;
}
if (Files.isRegularFile(linkToFile)) {
Files.copy(linkToFile, path, StandardCopyOption.REPLACE_EXISTING);
} else if (Files.isDirectory(linkToFile)) {
Files.delete(path);
MoreFiles.copyRecursively(linkToFile, path);
}
}
return FileVisitResult.CONTINUE;
}
};
Files.walkFileTree(destPath, copyDirVisitor);
}
// Disable the directory cache by default. Tests that want to enable it can call
// `enableDirCache` on this object. Only do this if a .buckconfig.local file does not already
// exist, however (we assume the test knows what it is doing at that point).
if (!Files.exists(getPath(".buckconfig.local"))) {
writeContentsToPath("[cache]\n mode =", ".buckconfig.local");
}
isSetUp = true;
return this;
}
TransactionLog(File tlogFile, Collection<String> globalStrings, boolean openExisting) {
boolean success = false;
try {
if (debug) {
log.debug(
"New TransactionLog file="
+ tlogFile
+ ", exists="
+ tlogFile.exists()
+ ", size="
+ tlogFile.length()
+ ", openExisting="
+ openExisting);
}
this.tlogFile = tlogFile;
raf = new RandomAccessFile(this.tlogFile, "rw");
long start = raf.length();
channel = raf.getChannel();
os = Channels.newOutputStream(channel);
fos = new FastOutputStream(os, new byte[65536], 0);
// fos = FastOutputStream.wrap(os);
if (openExisting) {
if (start > 0) {
readHeader(null);
raf.seek(start);
assert channel.position() == start;
fos.setWritten(start); // reflect that we aren't starting at the beginning
assert fos.size() == channel.size();
} else {
addGlobalStrings(globalStrings);
}
} else {
if (start > 0) {
log.warn("New transaction log already exists:" + tlogFile + " size=" + raf.length());
return;
}
if (start > 0) {
raf.setLength(0);
}
addGlobalStrings(globalStrings);
}
success = true;
assert ObjectReleaseTracker.track(this);
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
} finally {
if (!success && raf != null) {
try {
raf.close();
} catch (Exception e) {
log.error("Error closing tlog file (after error opening)", e);
}
}
}
}
@SuppressWarnings("deprecation") // uses internal test functionality.
@Override
protected void prepareWrite(WritableByteChannel channel) throws Exception {
dataFileWriter = new DataFileWriter<>(coder.createDatumWriter());
dataFileWriter.create(coder.getSchema(), Channels.newOutputStream(channel));
}
/**
* This will copy the template directory, renaming files named {@code foo.fixture} to {@code foo}
* in the process. Files whose names end in {@code .expected} will not be copied.
*/
@SuppressWarnings("PMD.EmptyCatchBlock")
public ProjectWorkspace setUp() throws IOException {
MoreFiles.copyRecursively(templatePath, destPath, BUILD_FILE_RENAME);
// Stamp the buck-out directory if it exists and isn't stamped already
try (OutputStream outputStream =
new BufferedOutputStream(
Channels.newOutputStream(
Files.newByteChannel(
destPath.resolve(BuckConstant.CURRENT_VERSION_FILE),
ImmutableSet.<OpenOption>of(
StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE))))) {
outputStream.write(BuckVersion.getVersion().getBytes(Charsets.UTF_8));
} catch (FileAlreadyExistsException | NoSuchFileException e) {
// If the current version file already exists we don't need to create it
// If buck-out doesn't exist we don't need to stamp it
}
// If there's a local.properties in the host project but not in the destination, make a copy.
Path localProperties = FileSystems.getDefault().getPath("local.properties");
Path destLocalProperties = destPath.resolve(localProperties.getFileName());
if (localProperties.toFile().exists() && !destLocalProperties.toFile().exists()) {
Files.copy(localProperties, destLocalProperties);
}
if (Platform.detect() == Platform.WINDOWS) {
// Hack for symlinks on Windows.
SimpleFileVisitor<Path> copyDirVisitor =
new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path path, BasicFileAttributes attrs)
throws IOException {
// On Windows, symbolic links from git repository are checked out as normal files
// containing a one-line path. In order to distinguish them, paths are read and
// pointed
// files are trued to locate. Once the pointed file is found, it will be copied to
// target.
// On NTFS length of path must be greater than 0 and less than 4096.
if (attrs.size() > 0 && attrs.size() <= 4096) {
String linkTo = new String(Files.readAllBytes(path), UTF_8);
Path linkToFile = null;
try {
linkToFile = templatePath.resolve(linkTo);
} catch (InvalidPathException e) {
// Let's assume we were reading a normal text file, and not something meant to be
// a
// link.
return FileVisitResult.CONTINUE;
}
if (Files.isRegularFile(linkToFile)) {
Files.copy(linkToFile, path, StandardCopyOption.REPLACE_EXISTING);
} else if (Files.isDirectory(linkToFile)) {
Files.delete(path);
MoreFiles.copyRecursively(linkToFile, path);
}
}
return FileVisitResult.CONTINUE;
}
};
Files.walkFileTree(destPath, copyDirVisitor);
}
isSetUp = true;
return this;
}
/**
* Returns an output stream based on a given writable byte channel.
*
* @param writableChannel The writable byte channel.
* @return An output stream based on a given writable byte channel.
*/
public static OutputStream getStream(WritableByteChannel writableChannel) {
return isBlocking(writableChannel)
? Channels.newOutputStream(writableChannel)
: new NbChannelOutputStream(writableChannel);
}
@Override
public OutputStream getOutputStream() throws IOException {
return Channels.newOutputStream(inner);
}
