public FileChunk get() throws IOException {
if (useLocalFile) {
startTime = System.currentTimeMillis();
finishTime = System.currentTimeMillis();
state = TajoProtos.FetcherState.FETCH_FINISHED;
return fileChunk;
}
this.startTime = System.currentTimeMillis();
this.state = TajoProtos.FetcherState.FETCH_FETCHING;
ChannelFuture future = null;
try {
future =
bootstrap
.clone()
.connect(new InetSocketAddress(host, port))
.addListener(ChannelFutureListener.CLOSE_ON_FAILURE);
// Wait until the connection attempt succeeds or fails.
Channel channel = future.awaitUninterruptibly().channel();
if (!future.isSuccess()) {
state = TajoProtos.FetcherState.FETCH_FAILED;
throw new IOException(future.cause());
}
String query = uri.getPath() + (uri.getRawQuery() != null ? "?" + uri.getRawQuery() : "");
// Prepare the HTTP request.
HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, query);
request.headers().set(HttpHeaders.Names.HOST, host);
request.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.CLOSE);
request.headers().set(HttpHeaders.Names.ACCEPT_ENCODING, HttpHeaders.Values.GZIP);
LOG.info("Status: " + getState() + ", URI:" + uri);
// Send the HTTP request.
channel.writeAndFlush(request);
// Wait for the server to close the connection. throw exception if failed
channel.closeFuture().syncUninterruptibly();
fileChunk.setLength(fileChunk.getFile().length());
return fileChunk;
} finally {
if (future != null && future.channel().isOpen()) {
// Close the channel to exit.
future.channel().close().awaitUninterruptibly();
}
this.finishTime = System.currentTimeMillis();
LOG.info(
"Fetcher finished:" + (finishTime - startTime) + " ms, " + getState() + ", URI:" + uri);
}
}
/**
* Flush the response contents to the output channel
*
* @param channel Channel to be used
*/
private void writeResponse(Channel channel) {
// Convert the response content to a ChannelBuffer.
ByteBuf buf = copiedBuffer(responseContent.toString(), CharsetUtil.UTF_8);
responseContent.setLength(0);
// Decide whether to close the connection or not.
boolean close =
HttpHeaders.Values.CLOSE.equalsIgnoreCase(request.headers().get(CONNECTION))
|| request.getProtocolVersion().equals(HttpVersion.HTTP_1_0)
&& !HttpHeaders.Values.KEEP_ALIVE.equalsIgnoreCase(
request.headers().get(CONNECTION));
// Build the response object.
FullHttpResponse response =
new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK, buf);
response.headers().set(CONTENT_TYPE, "application/json; charset=UTF-8");
if (!close) {
// There's no need to add 'Content-Length' header
// if this is the last response.
response.headers().set(CONTENT_LENGTH, buf.readableBytes());
}
try {
// Write the response.
ChannelFuture future = channel.writeAndFlush(response);
// Close the connection after the write operation is done if necessary.
if (close) {
future.addListener(ChannelFutureListener.CLOSE);
}
} catch (Exception e) {
logger.error("{}", e);
}
}
@Override
protected void initChannel(Channel channel) throws Exception {
ChannelPipeline pipeline = channel.pipeline();
int maxChunkSize = conf.getIntVar(TajoConf.ConfVars.SHUFFLE_FETCHER_CHUNK_MAX_SIZE);
int readTimeout = conf.getIntVar(TajoConf.ConfVars.SHUFFLE_FETCHER_READ_TIMEOUT);
pipeline.addLast("codec", new HttpClientCodec(4096, 8192, maxChunkSize));
pipeline.addLast("inflater", new HttpContentDecompressor());
pipeline.addLast("timeout", new ReadTimeoutHandler(readTimeout, TimeUnit.SECONDS));
pipeline.addLast("handler", new HttpClientHandler(file));
}
public void channelRead0(final ChannelHandlerContext ctx, final FullHttpRequest nettyRequest)
throws Exception {
if (!nettyRequest.getDecoderResult().isSuccess()) {
sendError(ctx, HttpResponseStatus.BAD_REQUEST);
return;
}
final long startTime = System.nanoTime();
final Request request =
new DefaultRequest(
new NettyHeadersBackedHeaders(nettyRequest.headers()),
nettyRequest.getMethod().name(),
nettyRequest.getUri(),
nettyRequest.content());
final Channel channel = ctx.channel();
final DefaultMutableStatus responseStatus = new DefaultMutableStatus();
final HttpHeaders httpHeaders = new DefaultHttpHeaders(false);
final MutableHeaders responseHeaders = new NettyHeadersBackedMutableHeaders(httpHeaders);
FileHttpTransmitter fileHttpTransmitter =
new DefaultFileHttpTransmitter(
nettyRequest,
httpHeaders,
channel,
compressResponses,
addResponseTimeHeader ? startTime : -1);
final DefaultEventController<RequestOutcome> requestOutcomeEventController =
new DefaultEventController<>();
// We own the lifecycle
nettyRequest.content().retain();
final Response response =
new DefaultResponse(
responseStatus,
responseHeaders,
fileHttpTransmitter,
ctx.alloc(),
new Action<ByteBuf>() {
@Override
public void execute(final ByteBuf byteBuf) throws Exception {
final HttpResponse nettyResponse =
new CustomHttpResponse(responseStatus.getResponseStatus(), httpHeaders);
nettyRequest.content().release();
responseHeaders.set(HttpHeaderConstants.CONTENT_LENGTH, byteBuf.writerIndex());
boolean shouldClose = true;
if (channel.isOpen()) {
if (isKeepAlive(nettyRequest)) {
responseHeaders.set(
HttpHeaderConstants.CONNECTION, HttpHeaderConstants.KEEP_ALIVE);
shouldClose = false;
}
long stopTime = System.nanoTime();
if (addResponseTimeHeader) {
responseHeaders.set(
"X-Response-Time", NumberUtil.toMillisDiffString(startTime, stopTime));
}
execController.getExecution().complete();
channel.writeAndFlush(nettyResponse);
channel.write(new DefaultHttpContent(byteBuf));
ChannelFuture future = channel.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
if (requestOutcomeEventController.isHasListeners()) {
Headers headers = new DelegatingHeaders(responseHeaders);
Status status =
new DefaultStatus(responseStatus.getCode(), responseStatus.getMessage());
SentResponse sentResponse = new DefaultSentResponse(headers, status);
RequestOutcome requestOutcome =
new DefaultRequestOutcome(
request, sentResponse, System.currentTimeMillis());
requestOutcomeEventController.fire(requestOutcome);
}
if (shouldClose) {
future.addListener(ChannelFutureListener.CLOSE);
}
}
}
});
InetSocketAddress socketAddress = (InetSocketAddress) channel.localAddress();
final BindAddress bindAddress = new InetSocketAddressBackedBindAddress(socketAddress);
Action<Action<Object>> subscribeHandler =
new Action<Action<Object>>() {
@Override
public void execute(Action<Object> thing) throws Exception {
channelSubscriptions.put(channel, thing);
channel
.closeFuture()
.addListener(
new ChannelFutureListener() {
@Override
public void operationComplete(ChannelFuture future) throws Exception {
channelSubscriptions.remove(channel);
}
});
}
};
final DirectChannelAccess directChannelAccess =
new DefaultDirectChannelAccess(channel, subscribeHandler);
execController.start(
new Action<Execution>() {
@Override
public void execute(Execution execution) throws Exception {
DefaultContext.RequestConstants requestConstants =
new DefaultContext.RequestConstants(
applicationConstants,
bindAddress,
request,
response,
directChannelAccess,
requestOutcomeEventController.getRegistry(),
execution);
new DefaultContext(requestConstants, registry, handlers, 0, return404).next();
}
});
}
public void channelRead0(final ChannelHandlerContext ctx, final FullHttpRequest nettyRequest)
throws Exception {
if (!nettyRequest.getDecoderResult().isSuccess()) {
sendError(ctx, HttpResponseStatus.BAD_REQUEST);
return;
}
final long startTime = addResponseTimeHeader ? System.nanoTime() : 0;
final Request request =
new DefaultRequest(
new NettyHeadersBackedHeaders(nettyRequest.headers()),
nettyRequest.getMethod().name(),
nettyRequest.getUri(),
nettyRequest.content());
final Channel channel = ctx.channel();
final DefaultMutableStatus responseStatus = new DefaultMutableStatus();
final HttpHeaders nettyHeaders = new DefaultHttpHeaders(false);
final MutableHeaders responseHeaders = new NettyHeadersBackedMutableHeaders(nettyHeaders);
final MimeTypes mimeTypes = registry.get(MimeTypes.class);
final DefaultEventController<RequestOutcome> requestOutcomeEventController =
new DefaultEventController<>();
final AtomicBoolean transmitted = new AtomicBoolean(false);
final ResponseTransmitter responseTransmitter =
new DefaultResponseTransmitter(
transmitted,
channel,
nettyRequest,
request,
nettyHeaders,
responseStatus,
requestOutcomeEventController,
startTime);
final Action<Action<? super ResponseTransmitter>> responseTransmitterWrapper =
Actions.wrap(responseTransmitter);
final FileHttpTransmitter fileHttpTransmitter =
new DefaultFileHttpTransmitter(
nettyHeaders,
mimeTypes,
compressResponses,
compressionMinSize,
compressionMimeTypeWhiteList,
compressionMimeTypeBlackList,
responseTransmitterWrapper);
StreamTransmitter streamTransmitter =
new DefaultStreamTransmitter(nettyRequest, nettyHeaders, channel);
final Response response =
new DefaultResponse(
responseStatus,
responseHeaders,
fileHttpTransmitter,
streamTransmitter,
ctx.alloc(),
new Action<ByteBuf>() {
@Override
public void execute(final ByteBuf byteBuf) throws Exception {
responseTransmitterWrapper.execute(
new Action<ResponseTransmitter>() {
@Override
public void execute(ResponseTransmitter responseTransmitter)
throws Exception {
nettyHeaders.set(HttpHeaders.Names.CONTENT_LENGTH, byteBuf.writerIndex());
responseTransmitter.transmit(new DefaultHttpContent(byteBuf));
}
});
}
});
InetSocketAddress socketAddress = (InetSocketAddress) channel.localAddress();
final BindAddress bindAddress = new InetSocketAddressBackedBindAddress(socketAddress);
Action<Action<Object>> subscribeHandler =
new Action<Action<Object>>() {
@Override
public void execute(Action<Object> thing) throws Exception {
transmitted.set(true);
channelSubscriptions.put(channel, thing);
channel
.closeFuture()
.addListener(
new ChannelFutureListener() {
@Override
public void operationComplete(ChannelFuture future) throws Exception {
channelSubscriptions.remove(channel);
}
});
}
};
final DirectChannelAccess directChannelAccess =
new DefaultDirectChannelAccess(channel, subscribeHandler);
final DefaultContext.RequestConstants requestConstants =
new DefaultContext.RequestConstants(
applicationConstants,
bindAddress,
request,
response,
directChannelAccess,
requestOutcomeEventController.getRegistry());
DefaultContext.start(
execController.getControl(),
requestConstants,
registry,
handlers,
return404,
new Action<Execution>() {
@Override
public void execute(Execution execution) throws Exception {
if (!transmitted.get()) {
Handler lastHandler = requestConstants.handler;
StringBuilder description = new StringBuilder();
description
.append("No response sent for ")
.append(request.getMethod().getName())
.append(" request to ")
.append(request.getUri())
.append(" (last handler: ");
if (lastHandler instanceof DescribingHandler) {
((DescribingHandler) lastHandler).describeTo(description);
} else {
DescribingHandlers.describeTo(lastHandler, description);
}
description.append(")");
String message = description.toString();
LOGGER.warn(message);
response.status(500);
if (launchConfig.isDevelopment()) {
response.send(message);
} else {
response.send();
}
}
}
});
}