private void doWalk(
Path path, FileStatusCallback callback, AtomicLong taskCount, SettableFuture<Void> future) {
try (SetThreadName ignored = new SetThreadName("HiveHdfsWalker")) {
RemoteIterator<LocatedFileStatus> iterator = getLocatedFileStatusRemoteIterator(path);
while (iterator.hasNext()) {
LocatedFileStatus status = getLocatedFileStatus(iterator);
// ignore hidden files. Hive ignores files starting with _ and . as well.
String fileName = status.getPath().getName();
if (fileName.startsWith("_") || fileName.startsWith(".")) {
continue;
}
if (isDirectory(status)) {
recursiveWalk(status.getPath(), callback, taskCount, future);
} else {
callback.process(status, status.getBlockLocations());
}
if (future.isDone()) {
return;
}
}
} catch (FileNotFoundException e) {
future.setException(new FileNotFoundException("Partition location does not exist: " + path));
} catch (Throwable t) {
future.setException(t);
} finally {
if (taskCount.decrementAndGet() == 0) {
future.set(null);
}
}
}
public boolean isRunning() {
if (hasStarted.isDone()) {
try {
TimeUnit.MILLISECONDS.sleep(500);
} catch (InterruptedException e) {
logger.trace("Exception while waiting for the proxy to confirm running status", e);
}
return !bossGroup.isShuttingDown() && !workerGroup.isShuttingDown();
} else {
return false;
}
}
/**
* Reserves the given number of bytes. Caller should wait on the returned future, before
* allocating more memory.
*/
public synchronized ListenableFuture<?> reserve(QueryId queryId, long bytes) {
checkArgument(bytes >= 0, "bytes is negative");
if (bytes != 0) {
queryMemoryReservations.merge(queryId, bytes, Long::sum);
}
freeBytes -= bytes;
if (freeBytes <= 0) {
if (future == null) {
future = SettableFuture.create();
}
checkState(!future.isDone(), "future is already completed");
return future;
}
return NOT_BLOCKED;
}
@Test
public void basicTimeoutTest() throws Exception {
// Tests various timeout scenarios
final SettableFuture<Void> serverConnection1Open = SettableFuture.create();
final SettableFuture<Void> clientConnection1Open = SettableFuture.create();
final SettableFuture<Void> serverConnection1Closed = SettableFuture.create();
final SettableFuture<Void> clientConnection1Closed = SettableFuture.create();
final SettableFuture<Void> serverConnection2Open = SettableFuture.create();
final SettableFuture<Void> clientConnection2Open = SettableFuture.create();
final SettableFuture<Void> serverConnection2Closed = SettableFuture.create();
final SettableFuture<Void> clientConnection2Closed = SettableFuture.create();
NioServer server =
new NioServer(
new StreamParserFactory() {
@Override
public ProtobufParser getNewParser(InetAddress inetAddress, int port) {
return new ProtobufParser<Protos.TwoWayChannelMessage>(
new ProtobufParser.Listener<Protos.TwoWayChannelMessage>() {
@Override
public void messageReceived(
ProtobufParser handler, Protos.TwoWayChannelMessage msg) {
fail.set(true);
}
@Override
public synchronized void connectionOpen(ProtobufParser handler) {
if (serverConnection1Open.isDone()) {
handler.setSocketTimeout(0);
serverConnection2Open.set(null);
} else serverConnection1Open.set(null);
}
@Override
public synchronized void connectionClosed(ProtobufParser handler) {
if (serverConnection1Closed.isDone()) {
serverConnection2Closed.set(null);
} else serverConnection1Closed.set(null);
}
},
Protos.TwoWayChannelMessage.getDefaultInstance(),
1000,
10);
}
},
new InetSocketAddress("localhost", 4243));
server.startAndWait();
openConnection(
new InetSocketAddress("localhost", 4243),
new ProtobufParser<Protos.TwoWayChannelMessage>(
new ProtobufParser.Listener<Protos.TwoWayChannelMessage>() {
@Override
public void messageReceived(ProtobufParser handler, Protos.TwoWayChannelMessage msg) {
fail.set(true);
}
@Override
public void connectionOpen(ProtobufParser handler) {
clientConnection1Open.set(null);
}
@Override
public void connectionClosed(ProtobufParser handler) {
clientConnection1Closed.set(null);
}
},
Protos.TwoWayChannelMessage.getDefaultInstance(),
1000,
0));
clientConnection1Open.get();
serverConnection1Open.get();
long closeDelayStart = System.currentTimeMillis();
clientConnection1Closed.get();
serverConnection1Closed.get();
long closeDelayFinish = System.currentTimeMillis();
ProtobufParser<Protos.TwoWayChannelMessage> client2Handler =
new ProtobufParser<Protos.TwoWayChannelMessage>(
new ProtobufParser.Listener<Protos.TwoWayChannelMessage>() {
@Override
public void messageReceived(ProtobufParser handler, Protos.TwoWayChannelMessage msg) {
fail.set(true);
}
@Override
public void connectionOpen(ProtobufParser handler) {
clientConnection2Open.set(null);
}
@Override
public void connectionClosed(ProtobufParser handler) {
clientConnection2Closed.set(null);
}
},
Protos.TwoWayChannelMessage.getDefaultInstance(),
1000,
0);
openConnection(new InetSocketAddress("localhost", 4243), client2Handler);
clientConnection2Open.get();
serverConnection2Open.get();
Thread.sleep((closeDelayFinish - closeDelayStart) * 10);
assertFalse(clientConnection2Closed.isDone() || serverConnection2Closed.isDone());
client2Handler.setSocketTimeout(10);
clientConnection2Closed.get();
serverConnection2Closed.get();
server.stopAndWait();
}
@Override
public boolean isDone() {
return settableFuture.isDone();
}