private ChannelFuture sendFile(ChannelHandlerContext ctx, Channel ch, FileChunk file)
throws IOException {
RandomAccessFile raf;
try {
raf = new RandomAccessFile(file.getFile(), "r");
} catch (FileNotFoundException fnfe) {
return null;
}
ChannelFuture writeFuture;
if (ch.getPipeline().get(SslHandler.class) != null) {
// Cannot use zero-copy with HTTPS.
writeFuture = ch.write(new ChunkedFile(raf, file.startOffset(), file.length(), 8192));
} else {
// No encryption - use zero-copy.
final FileRegion region =
new DefaultFileRegion(raf.getChannel(), file.startOffset(), file.length());
writeFuture = ch.write(region);
writeFuture.addListener(
new ChannelFutureListener() {
public void operationComplete(ChannelFuture future) {
region.releaseExternalResources();
}
});
}
return writeFuture;
}
public FileChunk get() throws IOException {
if (useLocalFile) {
startTime = System.currentTimeMillis();
finishTime = System.currentTimeMillis();
state = TajoProtos.FetcherState.FETCH_FINISHED;
return fileChunk;
}
this.startTime = System.currentTimeMillis();
this.state = TajoProtos.FetcherState.FETCH_FETCHING;
ChannelFuture future = null;
try {
future =
bootstrap
.clone()
.connect(new InetSocketAddress(host, port))
.addListener(ChannelFutureListener.CLOSE_ON_FAILURE);
// Wait until the connection attempt succeeds or fails.
Channel channel = future.awaitUninterruptibly().channel();
if (!future.isSuccess()) {
state = TajoProtos.FetcherState.FETCH_FAILED;
throw new IOException(future.cause());
}
String query = uri.getPath() + (uri.getRawQuery() != null ? "?" + uri.getRawQuery() : "");
// Prepare the HTTP request.
HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, query);
request.headers().set(HttpHeaders.Names.HOST, host);
request.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.CLOSE);
request.headers().set(HttpHeaders.Names.ACCEPT_ENCODING, HttpHeaders.Values.GZIP);
LOG.info("Status: " + getState() + ", URI:" + uri);
// Send the HTTP request.
channel.writeAndFlush(request);
// Wait for the server to close the connection. throw exception if failed
channel.closeFuture().syncUninterruptibly();
fileChunk.setLength(fileChunk.getFile().length());
return fileChunk;
} finally {
if (future != null && future.channel().isOpen()) {
// Close the channel to exit.
future.channel().close().awaitUninterruptibly();
}
this.finishTime = System.currentTimeMillis();
LOG.info(
"Fetcher finished:" + (finishTime - startTime) + " ms, " + getState() + ", URI:" + uri);
}
}
private ChannelFuture sendFile(ChannelHandlerContext ctx, Channel ch, FileChunk file)
throws IOException {
RandomAccessFile spill;
try {
spill = new RandomAccessFile(file.getFile(), "r");
} catch (FileNotFoundException e) {
LOG.info(file.getFile() + " not found");
return null;
}
ChannelFuture writeFuture;
if (ch.getPipeline().get(SslHandler.class) == null) {
final FadvisedFileRegion partition =
new FadvisedFileRegion(
spill,
file.startOffset,
file.length(),
manageOsCache,
readaheadLength,
readaheadPool,
file.getFile().getAbsolutePath());
writeFuture = ch.write(partition);
writeFuture.addListener(
new ChannelFutureListener() {
// TODO error handling; distinguish IO/connection failures,
// attribute to appropriate spill output
@Override
public void operationComplete(ChannelFuture future) {
partition.releaseExternalResources();
}
});
} else {
// HTTPS cannot be done with zero copy.
final FadvisedChunkedFile chunk =
new FadvisedChunkedFile(
spill,
file.startOffset,
file.length,
sslFileBufferSize,
manageOsCache,
readaheadLength,
readaheadPool,
file.getFile().getAbsolutePath());
writeFuture = ch.write(chunk);
}
metrics.shuffleConnections.incr();
metrics.shuffleOutputBytes.incr(file.length); // optimistic
return writeFuture;
}
public Fetcher(TajoConf conf, URI uri, FileChunk chunk) {
this.uri = uri;
this.fileChunk = chunk;
this.useLocalFile = !chunk.fromRemote();
this.state = TajoProtos.FetcherState.FETCH_INIT;
this.conf = conf;
String scheme = uri.getScheme() == null ? "http" : uri.getScheme();
this.host = uri.getHost() == null ? "localhost" : uri.getHost();
this.port = uri.getPort();
if (port == -1) {
if (scheme.equalsIgnoreCase("http")) {
this.port = 80;
} else if (scheme.equalsIgnoreCase("https")) {
this.port = 443;
}
}
if (!useLocalFile) {
bootstrap =
new Bootstrap()
.group(
NettyUtils.getSharedEventLoopGroup(
NettyUtils.GROUP.FETCHER,
conf.getIntVar(TajoConf.ConfVars.SHUFFLE_RPC_CLIENT_WORKER_THREAD_NUM)))
.channel(NioSocketChannel.class)
.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT)
.option(
ChannelOption.CONNECT_TIMEOUT_MILLIS,
conf.getIntVar(TajoConf.ConfVars.SHUFFLE_FETCHER_CONNECT_TIMEOUT) * 1000)
.option(ChannelOption.SO_RCVBUF, 1048576) // set 1M
.option(ChannelOption.TCP_NODELAY, true);
ChannelInitializer<Channel> initializer =
new HttpClientChannelInitializer(fileChunk.getFile());
bootstrap.handler(initializer);
}
}
@Override
public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
HttpRequest request = (HttpRequest) e.getMessage();
if (request.getMethod() != GET) {
sendError(ctx, METHOD_NOT_ALLOWED);
return;
}
// Parsing the URL into key-values
final Map<String, List<String>> params =
new QueryStringDecoder(request.getUri()).getParameters();
final List<String> types = params.get("type");
final List<String> taskIdList = params.get("ta");
final List<String> subQueryIds = params.get("sid");
final List<String> partitionIds = params.get("p");
if (types == null || taskIdList == null || subQueryIds == null || partitionIds == null) {
sendError(ctx, "Required type, taskIds, subquery Id, and partition id", BAD_REQUEST);
return;
}
if (types.size() != 1 || subQueryIds.size() != 1) {
sendError(ctx, "Required type, taskIds, subquery Id, and partition id", BAD_REQUEST);
return;
}
final List<FileChunk> chunks = Lists.newArrayList();
String repartitionType = types.get(0);
String sid = subQueryIds.get(0);
String partitionId = partitionIds.get(0);
List<String> taskIds = splitMaps(taskIdList);
// the working dir of tajo worker for each query
String queryBaseDir = queryId + "/output" + "/";
LOG.info(
"PullServer request param: repartitionType="
+ repartitionType
+ ", sid="
+ sid
+ ", partitionId="
+ partitionId
+ ", taskIds="
+ taskIdList);
String taskLocalDir = conf.get(ConfVars.WORKER_TEMPORAL_DIR.varname);
if (taskLocalDir == null || taskLocalDir.equals("")) {
LOG.error("Tajo local directory should be specified.");
}
LOG.info("PullServer baseDir: " + taskLocalDir + "/" + queryBaseDir);
// if a subquery requires a range partitioning
if (repartitionType.equals("r")) {
String ta = taskIds.get(0);
Path path =
localFS.makeQualified(
lDirAlloc.getLocalPathToRead(
queryBaseDir + "/" + sid + "/" + ta + "/output/", conf));
String startKey = params.get("start").get(0);
String endKey = params.get("end").get(0);
boolean last = params.get("final") != null;
FileChunk chunk;
try {
chunk = getFileCunks(path, startKey, endKey, last);
} catch (Throwable t) {
LOG.error("ERROR Request: " + request.getUri(), t);
sendError(ctx, "Cannot get file chunks to be sent", BAD_REQUEST);
return;
}
if (chunk != null) {
chunks.add(chunk);
}
// if a subquery requires a hash repartition
} else if (repartitionType.equals("h")) {
for (String ta : taskIds) {
Path path =
localFS.makeQualified(
lDirAlloc.getLocalPathToRead(
queryBaseDir + "/" + sid + "/" + ta + "/output/" + partitionId, conf));
File file = new File(path.toUri());
FileChunk chunk = new FileChunk(file, 0, file.length());
chunks.add(chunk);
}
} else {
LOG.error("Unknown repartition type: " + repartitionType);
return;
}
// Write the content.
Channel ch = e.getChannel();
if (chunks.size() == 0) {
HttpResponse response = new DefaultHttpResponse(HTTP_1_1, NO_CONTENT);
ch.write(response);
if (!isKeepAlive(request)) {
ch.close();
}
} else {
FileChunk[] file = chunks.toArray(new FileChunk[chunks.size()]);
HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
long totalSize = 0;
for (FileChunk chunk : file) {
totalSize += chunk.length();
}
setContentLength(response, totalSize);
// Write the initial line and the header.
ch.write(response);
ChannelFuture writeFuture = null;
for (FileChunk chunk : file) {
writeFuture = sendFile(ctx, ch, chunk);
if (writeFuture == null) {
sendError(ctx, NOT_FOUND);
return;
}
}
// Decide whether to close the connection or not.
if (!isKeepAlive(request)) {
// Close the connection when the whole content is written out.
writeFuture.addListener(ChannelFutureListener.CLOSE);
}
}
}
@Override
public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
HttpRequest request = (HttpRequest) e.getMessage();
if (request.getMethod() != GET) {
sendError(ctx, METHOD_NOT_ALLOWED);
return;
}
String base =
ContainerLocalizer.USERCACHE
+ "/"
+ userName
+ "/"
+ ContainerLocalizer.APPCACHE
+ "/"
+ appId
+ "/output"
+ "/";
final Map<String, List<String>> params =
new QueryStringDecoder(request.getUri()).getParameters();
List<FileChunk> chunks = Lists.newArrayList();
List<String> taskIds = splitMaps(params.get("ta"));
int sid = Integer.valueOf(params.get("sid").get(0));
int partitionId = Integer.valueOf(params.get("p").get(0));
for (String ta : taskIds) {
File file = new File(base + "/" + sid + "/" + ta + "/output/" + partitionId);
FileChunk chunk = new FileChunk(file, 0, file.length());
chunks.add(chunk);
}
FileChunk[] file = chunks.toArray(new FileChunk[chunks.size()]);
// try {
// file = retriever.handle(ctx, request);
// } catch (FileNotFoundException fnf) {
// LOG.error(fnf);
// sendError(ctx, NOT_FOUND);
// return;
// } catch (IllegalArgumentException iae) {
// LOG.error(iae);
// sendError(ctx, BAD_REQUEST);
// return;
// } catch (FileAccessForbiddenException fafe) {
// LOG.error(fafe);
// sendError(ctx, FORBIDDEN);
// return;
// } catch (IOException ioe) {
// LOG.error(ioe);
// sendError(ctx, INTERNAL_SERVER_ERROR);
// return;
// }
// Write the content.
Channel ch = e.getChannel();
if (file == null) {
HttpResponse response = new DefaultHttpResponse(HTTP_1_1, NO_CONTENT);
ch.write(response);
if (!isKeepAlive(request)) {
ch.close();
}
} else {
HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
long totalSize = 0;
for (FileChunk chunk : file) {
totalSize += chunk.length();
}
setContentLength(response, totalSize);
// Write the initial line and the header.
ch.write(response);
ChannelFuture writeFuture = null;
for (FileChunk chunk : file) {
writeFuture = sendFile(ctx, ch, chunk);
if (writeFuture == null) {
sendError(ctx, NOT_FOUND);
return;
}
}
// Decide whether to close the connection or not.
if (!isKeepAlive(request)) {
// Close the connection when the whole content is written out.
writeFuture.addListener(ChannelFutureListener.CLOSE);
}
}
}