public static void removePluggableHandlers(ChannelPipeline pipeline) {
for (String name : pipeline.getNames()) {
if (name.startsWith("pluggable_")) {
pipeline.remove(name);
}
}
}
private static void ensureThatExceptionHandlerIsLast(ChannelPipeline pipeline) {
ChannelInboundHandler exceptionHandler = ChannelExceptionHandler.getInstance();
if (pipeline.last() != exceptionHandler || pipeline.context(exceptionHandler) == null) {
return;
}
pipeline.remove(exceptionHandler);
pipeline.addLast(exceptionHandler);
}
@Override
public ChannelPipeline getPipeline() throws Exception {
ChannelPipeline pipeline = pipeline();
pipeline.addLast("codec", new HttpClientCodec());
pipeline.addLast("inflater", new HttpContentDecompressor());
pipeline.addLast("handler", new HttpClientHandler(file));
return pipeline;
}
@Override
public void initChannel(SocketChannel ch) throws Exception {
ChannelPipeline pipeline = ch.pipeline();
pipeline.addLast("http-codec", new HttpServerCodec());
// pipeline.addLast("log", new LoggingHandler(LogLevel.INFO));
pipeline.addLast("handler", new RFRequestHandler());
}
/**
* Shutdown this client and close all open connections. The client should be discarded after
* calling shutdown.
*/
public void shutdown() {
for (Channel c : channels) {
ChannelPipeline pipeline = c.getPipeline();
RedisAsyncConnection<?, ?> connection = pipeline.get(RedisAsyncConnection.class);
connection.close();
}
ChannelGroupFuture future = channels.close();
future.awaitUninterruptibly();
bootstrap.releaseExternalResources();
}
@Override
public void initChannel(SocketChannel ch) throws Exception {
ChannelPipeline pipeline = ch.pipeline();
SSLContext sslContext = Utils.createSSLContext();
SSLEngine engine = sslContext.createSSLEngine();
engine.setUseClientMode(true);
pipeline.addLast("ssl", new SslHandler(engine));
pipeline.addLast("framer", new LineBasedFrameDecoder(1000 * 1000 * 10, true, false));
pipeline.addLast("decoder", new StringDecoder(CharsetUtil.UTF_8));
pipeline.addLast("encoder", new StringEncoder(CharsetUtil.UTF_8));
pipeline.addLast("handler", connection);
}
@Override
protected void initChannel(Channel channel) throws Exception {
ChannelPipeline pipeline = channel.pipeline();
int maxChunkSize = conf.getIntVar(TajoConf.ConfVars.SHUFFLE_FETCHER_CHUNK_MAX_SIZE);
int readTimeout = conf.getIntVar(TajoConf.ConfVars.SHUFFLE_FETCHER_READ_TIMEOUT);
pipeline.addLast("codec", new HttpClientCodec(4096, 8192, maxChunkSize));
pipeline.addLast("inflater", new HttpContentDecompressor());
pipeline.addLast("timeout", new ReadTimeoutHandler(readTimeout, TimeUnit.SECONDS));
pipeline.addLast("handler", new HttpClientHandler(file));
}
private static String getWebSocketLocation(ChannelPipeline cp, HttpRequest req, String path) {
String protocol = "ws";
if (cp.get(SslHandler.class) != null) {
// SSL in use so use Secure WebSockets
protocol = "wss";
}
return protocol + "://" + req.getHttpHeaders().getHeaderString(HttpHeaders.Names.HOST) + path;
}
@Override
public ChannelPipeline getPipeline() throws Exception {
ChannelPipeline pipeline =
pipeline(
new HttpRequestDecoder(),
new HttpChunkAggregator(1048576),
new HttpResponseEncoder());
for (Consumer<ChannelPipeline> consumer : pipelineConsumers.compute()) {
try {
consumer.consume(pipeline);
} catch (Throwable e) {
LOG.error(e);
}
}
pipeline.addLast("defaultHandler", defaultHandler);
return pipeline;
}
@Override
protected void channelRead0(ChannelHandlerContext context, ByteBuf message) throws Exception {
ByteBuf buffer = getBufferIfSufficient(message, UUID_LENGTH, context);
if (buffer == null) {
message.release();
} else {
UUID uuid = new UUID(buffer.readLong(), buffer.readLong());
for (BinaryRequestHandler customHandler : BinaryRequestHandler.EP_NAME.getExtensions()) {
if (uuid.equals(customHandler.getId())) {
ChannelPipeline pipeline = context.pipeline();
pipeline.addLast(customHandler.getInboundHandler());
ensureThatExceptionHandlerIsLast(pipeline);
pipeline.remove(this);
context.fireChannelRead(buffer);
break;
}
}
}
}
/**
* Reconnect to the remote address that the closed channel was connected to. This creates a new
* {@link ChannelPipeline} with the same handler instances contained in the old channel's
* pipeline.
*
* @param timeout Timer task handle.
* @throws Exception when reconnection fails.
*/
@Override
public void run(Timeout timeout) throws Exception {
ChannelPipeline old = channel.pipeline();
final CommandHandler<?, ?> handler = old.get(CommandHandler.class);
final RedisAsyncConnection<?, ?> connection = old.get(RedisAsyncConnection.class);
ChannelFuture connect = null;
// TODO use better concurrent workaround
synchronized (bootstrap) {
connect =
bootstrap
.handler(
new ChannelInitializer<Channel>() {
@Override
protected void initChannel(Channel ch) throws Exception {
ch.pipeline().addLast(this, handler, connection);
}
})
.connect();
}
connect.sync();
}
protected void decode(ChannelHandlerContext context, ByteBuf buffer) throws Exception {
ChannelPipeline pipeline = context.pipeline();
if (detectSsl && SslHandler.isEncrypted(buffer)) {
SSLEngine engine = SSL_SERVER_CONTEXT.getValue().createSSLEngine();
engine.setUseClientMode(false);
pipeline.addLast(
new SslHandler(engine),
new ChunkedWriteHandler(),
new PortUnificationServerHandler(delegatingHttpRequestHandler, false, detectGzip));
} else {
int magic1 = buffer.getUnsignedByte(buffer.readerIndex());
int magic2 = buffer.getUnsignedByte(buffer.readerIndex() + 1);
if (detectGzip && magic1 == 31 && magic2 == 139) {
pipeline.addLast(
new JZlibEncoder(ZlibWrapper.GZIP),
new JdkZlibDecoder(ZlibWrapper.GZIP),
new PortUnificationServerHandler(delegatingHttpRequestHandler, detectSsl, false));
} else if (isHttp(magic1, magic2)) {
NettyUtil.initHttpHandlers(pipeline);
pipeline.addLast(delegatingHttpRequestHandler);
if (BuiltInServer.LOG.isDebugEnabled()) {
pipeline.addLast(
new ChannelOutboundHandlerAdapter() {
@Override
public void write(
ChannelHandlerContext context, Object message, ChannelPromise promise)
throws Exception {
if (message instanceof HttpResponse) {
// BuiltInServer.LOG.debug("OUT HTTP:\n" + message);
HttpResponse response = (HttpResponse) message;
BuiltInServer.LOG.debug(
"OUT HTTP: "
+ response.getStatus().code()
+ " "
+ response.headers().get("Content-type"));
}
super.write(context, message, promise);
}
});
}
} else if (magic1 == 'C' && magic2 == 'H') {
buffer.skipBytes(2);
pipeline.addLast(new CustomHandlerDelegator());
} else {
BuiltInServer.LOG.warn("unknown request, first two bytes " + magic1 + " " + magic2);
context.close();
}
}
// must be after new channels handlers addition (netty bug?)
ensureThatExceptionHandlerIsLast(pipeline);
pipeline.remove(this);
context.fireChannelRead(buffer);
}
@Override
public ChannelPipeline getPipeline() throws Exception {
ChannelPipeline pipeline = Channels.pipeline();
pipeline.addLast("openChannels", transport.serverOpenChannels);
HttpRequestDecoder requestDecoder =
new HttpRequestDecoder(
(int) transport.maxInitialLineLength.bytes(),
(int) transport.maxHeaderSize.bytes(),
(int) transport.maxChunkSize.bytes());
if (transport.maxCumulationBufferCapacity != null) {
if (transport.maxCumulationBufferCapacity.bytes() > Integer.MAX_VALUE) {
requestDecoder.setMaxCumulationBufferCapacity(Integer.MAX_VALUE);
} else {
requestDecoder.setMaxCumulationBufferCapacity(
(int) transport.maxCumulationBufferCapacity.bytes());
}
}
if (transport.maxCompositeBufferComponents != -1) {
requestDecoder.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents);
}
pipeline.addLast("decoder", requestDecoder);
pipeline.addLast("decoder_compress", new ESHttpContentDecompressor(transport.compression));
HttpChunkAggregator httpChunkAggregator =
new HttpChunkAggregator((int) transport.maxContentLength.bytes());
if (transport.maxCompositeBufferComponents != -1) {
httpChunkAggregator.setMaxCumulationBufferComponents(
transport.maxCompositeBufferComponents);
}
pipeline.addLast("aggregator", httpChunkAggregator);
pipeline.addLast("encoder", new ESHttpResponseEncoder());
if (transport.compression) {
pipeline.addLast("encoder_compress", new HttpContentCompressor(transport.compressionLevel));
}
if (transport.pipelining) {
pipeline.addLast("pipelining", new HttpPipeliningHandler(transport.pipeliningMaxEvents));
}
pipeline.addLast("handler", requestHandler);
return pipeline;
}
@Override
public ChannelPipeline getPipeline() throws Exception {
ChannelPipeline pipeline = Channels.pipeline();
if (sslFactory != null) {
pipeline.addLast("ssl", new SslHandler(sslFactory.createSSLEngine()));
}
pipeline.addLast("decoder", new HttpRequestDecoder());
pipeline.addLast("aggregator", new HttpChunkAggregator(1 << 16));
pipeline.addLast("encoder", new HttpResponseEncoder());
pipeline.addLast("chunking", new ChunkedWriteHandler());
pipeline.addLast("shuffle", PullServer);
return pipeline;
// TODO factor security manager into pipeline
// TODO factor out encode/decode to permit binary shuffle
// TODO factor out decode of index to permit alt. models
}
@Override
protected void initChannel(Channel ch) throws Exception {
ChannelPipeline pipeline = ch.pipeline();
pipeline.addLast(new HttpClientCodec());
pipeline.addLast(new HttpObjectAggregator(Integer.MAX_VALUE));
}
NetSocket createNetSocket() {
DefaultNetSocket socket =
new DefaultNetSocket(vertx, channel, context, server.tcpHelper, false);
Map<Channel, DefaultNetSocket> connectionMap = new HashMap<Channel, DefaultNetSocket>(1);
connectionMap.put(channel, socket);
// Flush out all pending data
endReadAndFlush();
// remove old http handlers and replace the old handler with one that handle plain sockets
ChannelPipeline pipeline = channel.pipeline();
ChannelHandler compressor = pipeline.get(HttpChunkContentCompressor.class);
if (compressor != null) {
pipeline.remove(compressor);
}
pipeline.remove("httpDecoder");
if (pipeline.get("chunkedWriter") != null) {
pipeline.remove("chunkedWriter");
}
channel
.pipeline()
.replace(
"handler",
"handler",
new VertxNetHandler(server.vertx, connectionMap) {
@Override
public void exceptionCaught(ChannelHandlerContext chctx, Throwable t)
throws Exception {
// remove from the real mapping
server.connectionMap.remove(channel);
super.exceptionCaught(chctx, t);
}
@Override
public void channelInactive(ChannelHandlerContext chctx) throws Exception {
// remove from the real mapping
server.connectionMap.remove(channel);
super.channelInactive(chctx);
}
@Override
public void channelRead(ChannelHandlerContext chctx, Object msg) throws Exception {
if (msg instanceof HttpContent) {
ReferenceCountUtil.release(msg);
return;
}
super.channelRead(chctx, msg);
}
});
// check if the encoder can be removed yet or not.
if (lastWriteFuture == null) {
channel.pipeline().remove("httpEncoder");
} else {
lastWriteFuture.addListener(
new ChannelFutureListener() {
@Override
public void operationComplete(ChannelFuture future) throws Exception {
channel.pipeline().remove("httpEncoder");
}
});
}
return socket;
}
public void checkPipeline(String protocol, ChannelPipeline pipeline, ChannelBuffer buf)
throws Exception {
Object tmp = buf.duplicate();
// Frame decoder
FrameDecoder frameDecoder = (FrameDecoder) pipeline.get("frameDecoder");
if (frameDecoder != null) {
try {
Method method =
frameDecoder
.getClass()
.getDeclaredMethod(
"decode", ChannelHandlerContext.class, Channel.class, ChannelBuffer.class);
method.setAccessible(true);
tmp = method.invoke(frameDecoder, null, null, tmp);
} catch (NoSuchMethodException error) {
Method method =
frameDecoder
.getClass()
.getSuperclass()
.getDeclaredMethod(
"decode", ChannelHandlerContext.class, Channel.class, ChannelBuffer.class);
method.setAccessible(true);
tmp = method.invoke(frameDecoder, null, null, tmp);
}
}
// String decoder
if (pipeline.get("stringDecoder") != null) {
StringDecoder stringDecoder = new StringDecoder();
if (tmp != null) {
try {
Method method =
stringDecoder
.getClass()
.getDeclaredMethod(
"decode", ChannelHandlerContext.class, Channel.class, Object.class);
method.setAccessible(true);
tmp = method.invoke(stringDecoder, null, null, tmp);
} catch (NoSuchMethodException error) {
Method method =
stringDecoder
.getClass()
.getSuperclass()
.getDeclaredMethod(
"decode", ChannelHandlerContext.class, Channel.class, Object.class);
method.setAccessible(true);
tmp = method.invoke(stringDecoder, null, null, tmp);
}
}
}
// Protocol decoder
BaseProtocolDecoder protocolDecoder = (BaseProtocolDecoder) pipeline.get("objectDecoder");
if (tmp != null) {
try {
Method method =
protocolDecoder
.getClass()
.getDeclaredMethod(
"decode",
ChannelHandlerContext.class,
Channel.class,
SocketAddress.class,
Object.class);
method.setAccessible(true);
tmp = method.invoke(protocolDecoder, null, null, null, tmp);
} catch (NoSuchMethodException error) {
Method method =
protocolDecoder
.getClass()
.getSuperclass()
.getDeclaredMethod(
"decode",
ChannelHandlerContext.class,
Channel.class,
SocketAddress.class,
Object.class);
method.setAccessible(true);
tmp = method.invoke(protocolDecoder, null, null, null, tmp);
}
}
if (tmp != null) {
Log.info("Protocol " + protocol + " possible match");
} else if (showFailed) {
Log.info("Protocol " + protocol + " no match");
}
}
private void websocketHandshake(final ChannelHandlerContext ctx, HttpRequest req, MessageEvent e)
throws Exception {
// Create the WebSocket handshake response.
HttpResponse res =
new DefaultHttpResponse(
HttpVersion.HTTP_1_1, new HttpResponseStatus(101, "Web Socket Protocol Handshake"));
res.addHeader(HttpHeaders.Names.UPGRADE, HttpHeaders.Values.WEBSOCKET);
res.addHeader(CONNECTION, HttpHeaders.Values.UPGRADE);
// Fill in the headers and contents depending on handshake method.
if (req.containsHeader(SEC_WEBSOCKET_KEY1) && req.containsHeader(SEC_WEBSOCKET_KEY2)) {
// New handshake method with a challenge:
res.addHeader(SEC_WEBSOCKET_ORIGIN, req.getHeader(ORIGIN));
res.addHeader(
SEC_WEBSOCKET_LOCATION, "ws://" + req.getHeader(HttpHeaders.Names.HOST) + req.getUri());
String protocol = req.getHeader(SEC_WEBSOCKET_PROTOCOL);
if (protocol != null) {
res.addHeader(SEC_WEBSOCKET_PROTOCOL, protocol);
}
// Calculate the answer of the challenge.
String key1 = req.getHeader(SEC_WEBSOCKET_KEY1);
String key2 = req.getHeader(SEC_WEBSOCKET_KEY2);
int a =
(int)
(Long.parseLong(key1.replaceAll("[^0-9]", ""))
/ key1.replaceAll("[^ ]", "").length());
int b =
(int)
(Long.parseLong(key2.replaceAll("[^0-9]", ""))
/ key2.replaceAll("[^ ]", "").length());
long c = req.getContent().readLong();
ChannelBuffer input = ChannelBuffers.buffer(16);
input.writeInt(a);
input.writeInt(b);
input.writeLong(c);
try {
ChannelBuffer output =
ChannelBuffers.wrappedBuffer(MessageDigest.getInstance("MD5").digest(input.array()));
res.setContent(output);
} catch (NoSuchAlgorithmException ex) {
throw new UnexpectedException(ex);
}
} else {
// Old handshake method with no challenge:
res.addHeader(WEBSOCKET_ORIGIN, req.getHeader(ORIGIN));
res.addHeader(
WEBSOCKET_LOCATION, "ws://" + req.getHeader(HttpHeaders.Names.HOST) + req.getUri());
String protocol = req.getHeader(WEBSOCKET_PROTOCOL);
if (protocol != null) {
res.addHeader(WEBSOCKET_PROTOCOL, protocol);
}
}
// Keep the original request
Http.Request request = parseRequest(ctx, req);
// Route the websocket request
request.method = "WS";
Map<String, String> route = Router.route(request.method, request.path);
if (!route.containsKey("action")) {
// No route found to handle this websocket connection
ctx.getChannel()
.write(new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.NOT_FOUND));
return;
}
// Upgrade the connection and send the handshake response.
ChannelPipeline p = ctx.getChannel().getPipeline();
p.remove("aggregator");
p.replace("decoder", "wsdecoder", new WebSocketFrameDecoder());
// Connect
ctx.getChannel().write(res);
p.replace("encoder", "wsencoder", new WebSocketFrameEncoder());
req.setMethod(new HttpMethod("WEBSOCKET"));
// Inbound
Http.Inbound inbound =
new Http.Inbound() {
@Override
public boolean isOpen() {
return ctx.getChannel().isOpen();
}
};
channels.put(ctx, inbound);
// Outbound
Http.Outbound outbound =
new Http.Outbound() {
final List<ChannelFuture> writeFutures =
Collections.synchronizedList(new ArrayList<ChannelFuture>());
Promise<Void> closeTask;
synchronized void writeAndClose(ChannelFuture writeFuture) {
if (!writeFuture.isDone()) {
writeFutures.add(writeFuture);
writeFuture.addListener(
new ChannelFutureListener() {
public void operationComplete(ChannelFuture cf) throws Exception {
writeFutures.remove(cf);
futureClose();
}
});
}
}
void futureClose() {
if (closeTask != null && writeFutures.isEmpty()) {
closeTask.invoke(null);
}
}
@Override
public void send(String data) {
if (!isOpen()) {
throw new IllegalStateException("The outbound channel is closed");
}
writeAndClose(ctx.getChannel().write(new DefaultWebSocketFrame(data)));
}
@Override
public void send(byte opcode, byte[] data, int offset, int length) {
if (!isOpen()) {
throw new IllegalStateException("The outbound channel is closed");
}
writeAndClose(
ctx.getChannel()
.write(new DefaultWebSocketFrame(opcode, wrappedBuffer(data, offset, length))));
}
@Override
public synchronized boolean isOpen() {
return ctx.getChannel().isOpen() && closeTask == null;
}
@Override
public synchronized void close() {
closeTask = new Promise<Void>();
closeTask.onRedeem(
new Action<Promise<Void>>() {
public void invoke(Promise<Void> completed) {
writeFutures.clear();
ctx.getChannel().disconnect();
closeTask = null;
}
});
futureClose();
}
};
Invoker.invoke(new WebSocketInvocation(route, request, inbound, outbound, ctx, e));
}
@Override
protected void initChannel(Channel ch) throws Exception {
ChannelPipeline pipeline = ch.pipeline();
pipeline.addLast(new LineBasedFrameDecoder(65 * 1024));
pipeline.addLast(new FrameHandler());
}
@Override
public final boolean trySend(HttpResponse message) {
try {
final HttpRequestWrapper nettyRequest = (HttpRequestWrapper) message.getRequest();
final FullHttpRequest req = nettyRequest.req;
final ChannelHandlerContext ctx = nettyRequest.ctx;
final HttpResponseStatus status = HttpResponseStatus.valueOf(message.getStatus());
if (message.getStatus() >= 400 && message.getStatus() < 600) {
sendHttpResponse(
ctx, req, new DefaultFullHttpResponse(req.getProtocolVersion(), status), false);
close();
return true;
}
if (message.getRedirectPath() != null) {
sendHttpRedirect(ctx, req, message.getRedirectPath());
close();
return true;
}
FullHttpResponse res;
if (message.getStringBody() != null)
res =
new DefaultFullHttpResponse(
req.getProtocolVersion(),
status,
Unpooled.wrappedBuffer(message.getStringBody().getBytes()));
else if (message.getByteBufferBody() != null)
res =
new DefaultFullHttpResponse(
req.getProtocolVersion(),
status,
Unpooled.wrappedBuffer(message.getByteBufferBody()));
else res = new DefaultFullHttpResponse(req.getProtocolVersion(), status);
if (message.getCookies() != null) {
final ServerCookieEncoder enc = ServerCookieEncoder.STRICT;
for (final Cookie c : message.getCookies())
HttpHeaders.setHeader(res, COOKIE, enc.encode(getNettyCookie(c)));
}
if (message.getHeaders() != null) {
for (final Map.Entry<String, String> h : message.getHeaders().entries())
HttpHeaders.setHeader(res, h.getKey(), h.getValue());
}
if (message.getContentType() != null) {
String ct = message.getContentType();
if (message.getCharacterEncoding() != null)
ct = ct + "; charset=" + message.getCharacterEncoding().name();
HttpHeaders.setHeader(res, CONTENT_TYPE, ct);
}
final boolean sseStarted = message.shouldStartActor();
if (trackSession(sseStarted)) {
final String sessionId = UUID.randomUUID().toString();
res.headers()
.add(SET_COOKIE, ServerCookieEncoder.STRICT.encode(SESSION_COOKIE_KEY, sessionId));
startSession(sessionId, actorContext);
}
if (!sseStarted) {
final String stringBody = message.getStringBody();
long contentLength = 0L;
if (stringBody != null) contentLength = stringBody.getBytes().length;
else {
final ByteBuffer byteBufferBody = message.getByteBufferBody();
if (byteBufferBody != null) contentLength = byteBufferBody.remaining();
}
res.headers().add(CONTENT_LENGTH, contentLength);
}
final HttpStreamActorAdapter httpStreamActorAdapter;
if (sseStarted)
// This will copy the request content, which must still be referenceable, doing before the
// request handler
// unallocates it (unfortunately it is explicitly reference-counted in Netty)
httpStreamActorAdapter = new HttpStreamActorAdapter(ctx, req);
else httpStreamActorAdapter = null;
sendHttpResponse(ctx, req, res, false);
if (sseStarted) {
if (httpResponseEncoderName != null) {
ctx.pipeline().remove(httpResponseEncoderName);
} else {
final ChannelPipeline pl = ctx.pipeline();
final List<String> handlerKeysToBeRemoved = new ArrayList<>();
for (final Map.Entry<String, ChannelHandler> e : pl) {
if (e.getValue() instanceof HttpResponseEncoder)
handlerKeysToBeRemoved.add(e.getKey());
}
for (final String k : handlerKeysToBeRemoved) pl.remove(k);
}
try {
message.getFrom().send(new HttpStreamOpened(httpStreamActorAdapter.ref(), message));
} catch (SuspendExecution e) {
throw new AssertionError(e);
}
}
return true;
} finally {
if (actor != null) actor.unwatch();
}
}
