private void wrap(ChannelHandlerContext ctx, boolean inUnwrap) throws SSLException {
ByteBuf out = null;
ChannelPromise promise = null;
ByteBufAllocator alloc = ctx.alloc();
try {
for (; ; ) {
Object msg = pendingUnencryptedWrites.current();
if (msg == null) {
break;
}
ByteBuf buf = (ByteBuf) msg;
if (out == null) {
out = allocateOutNetBuf(ctx, buf.readableBytes());
}
SSLEngineResult result = wrap(alloc, engine, buf, out);
if (!buf.isReadable()) {
promise = pendingUnencryptedWrites.remove();
} else {
promise = null;
}
if (result.getStatus() == Status.CLOSED) {
// SSLEngine has been closed already.
// Any further write attempts should be denied.
pendingUnencryptedWrites.removeAndFailAll(SSLENGINE_CLOSED);
return;
} else {
switch (result.getHandshakeStatus()) {
case NEED_TASK:
runDelegatedTasks();
break;
case FINISHED:
setHandshakeSuccess();
// deliberate fall-through
case NOT_HANDSHAKING:
setHandshakeSuccessIfStillHandshaking();
// deliberate fall-through
case NEED_WRAP:
finishWrap(ctx, out, promise, inUnwrap);
promise = null;
out = null;
break;
case NEED_UNWRAP:
return;
default:
throw new IllegalStateException(
"Unknown handshake status: " + result.getHandshakeStatus());
}
}
}
} catch (SSLException e) {
setHandshakeFailure(ctx, e);
throw e;
} finally {
finishWrap(ctx, out, promise, inUnwrap);
}
}
private void wrapNonAppData(ChannelHandlerContext ctx, boolean inUnwrap) throws SSLException {
ByteBuf out = null;
try {
for (; ; ) {
if (out == null) {
out = ctx.alloc().buffer(maxPacketBufferSize);
}
SSLEngineResult result = wrap(engine, Unpooled.EMPTY_BUFFER, out);
if (result.bytesProduced() > 0) {
ctx.write(out);
if (inUnwrap) {
needsFlush = true;
}
out = null;
}
switch (result.getHandshakeStatus()) {
case FINISHED:
setHandshakeSuccess();
break;
case NEED_TASK:
runDelegatedTasks();
break;
case NEED_UNWRAP:
if (!inUnwrap) {
unwrapNonApp(ctx);
}
break;
case NEED_WRAP:
break;
case NOT_HANDSHAKING:
setHandshakeSuccessIfStillHandshaking();
// Workaround for TLS False Start problem reported at:
// https://github.com/netty/netty/issues/1108#issuecomment-14266970
if (!inUnwrap) {
unwrapNonApp(ctx);
}
break;
default:
throw new IllegalStateException(
"Unknown handshake status: " + result.getHandshakeStatus());
}
if (result.bytesProduced() == 0) {
break;
}
}
} catch (SSLException e) {
setHandshakeFailure(e);
throw e;
} finally {
if (out != null) {
out.release();
}
}
}
/** Unwraps inbound SSL records. */
private boolean unwrap(ChannelHandlerContext ctx, ByteBuf packet, int offset, int length)
throws SSLException {
boolean decoded = false;
boolean wrapLater = false;
boolean notifyClosure = false;
ByteBuf decodeOut = allocate(ctx, length);
try {
for (; ; ) {
final SSLEngineResult result = unwrap(engine, packet, offset, length, decodeOut);
final Status status = result.getStatus();
final HandshakeStatus handshakeStatus = result.getHandshakeStatus();
final int produced = result.bytesProduced();
final int consumed = result.bytesConsumed();
// Update indexes for the next iteration
offset += consumed;
length -= consumed;
switch (status) {
case BUFFER_OVERFLOW:
int readableBytes = decodeOut.readableBytes();
if (readableBytes > 0) {
decoded = true;
ctx.fireChannelRead(decodeOut);
} else {
decodeOut.release();
}
// Allocate a new buffer which can hold all the rest data and loop again.
// TODO: We may want to reconsider how we calculate the length here as we may
// have more then one ssl message to decode.
decodeOut =
allocate(ctx, engine.getSession().getApplicationBufferSize() - readableBytes);
continue;
case CLOSED:
// notify about the CLOSED state of the SSLEngine. See #137
notifyClosure = true;
break;
default:
break;
}
switch (handshakeStatus) {
case NEED_UNWRAP:
break;
case NEED_WRAP:
wrapNonAppData(ctx, true);
break;
case NEED_TASK:
runDelegatedTasks();
break;
case FINISHED:
setHandshakeSuccess();
wrapLater = true;
continue;
case NOT_HANDSHAKING:
if (setHandshakeSuccessIfStillHandshaking()) {
wrapLater = true;
continue;
}
if (flushedBeforeHandshake) {
// We need to call wrap(...) in case there was a flush done before the handshake
// completed.
//
// See https://github.com/netty/netty/pull/2437
flushedBeforeHandshake = false;
wrapLater = true;
}
break;
default:
throw new IllegalStateException("unknown handshake status: " + handshakeStatus);
}
if (status == Status.BUFFER_UNDERFLOW || consumed == 0 && produced == 0) {
break;
}
}
if (wrapLater) {
wrap(ctx, true);
}
if (notifyClosure) {
sslCloseFuture.trySuccess(ctx.channel());
}
} catch (SSLException e) {
setHandshakeFailure(ctx, e);
throw e;
} finally {
if (decodeOut.isReadable()) {
decoded = true;
ctx.fireChannelRead(decodeOut);
} else {
decodeOut.release();
}
}
return decoded;
}
private void wrapNonAppData(ChannelHandlerContext ctx, boolean inUnwrap) throws SSLException {
ByteBuf out = null;
ByteBufAllocator alloc = ctx.alloc();
try {
for (; ; ) {
if (out == null) {
out = allocateOutNetBuf(ctx, 0);
}
SSLEngineResult result = wrap(alloc, engine, Unpooled.EMPTY_BUFFER, out);
if (result.bytesProduced() > 0) {
ctx.write(out);
if (inUnwrap) {
needsFlush = true;
}
out = null;
}
switch (result.getHandshakeStatus()) {
case FINISHED:
setHandshakeSuccess();
break;
case NEED_TASK:
runDelegatedTasks();
break;
case NEED_UNWRAP:
if (!inUnwrap) {
unwrapNonAppData(ctx);
}
break;
case NEED_WRAP:
break;
case NOT_HANDSHAKING:
setHandshakeSuccessIfStillHandshaking();
// Workaround for TLS False Start problem reported at:
// https://github.com/netty/netty/issues/1108#issuecomment-14266970
if (!inUnwrap) {
unwrapNonAppData(ctx);
}
break;
default:
throw new IllegalStateException(
"Unknown handshake status: " + result.getHandshakeStatus());
}
if (result.bytesProduced() == 0) {
break;
}
// It should not consume empty buffers when it is not handshaking
// Fix for Android, where it was encrypting empty buffers even when not handshaking
if (result.bytesConsumed() == 0
&& result.getHandshakeStatus() == HandshakeStatus.NOT_HANDSHAKING) {
break;
}
}
} catch (SSLException e) {
setHandshakeFailure(ctx, e);
throw e;
} finally {
if (out != null) {
out.release();
}
}
}
@Override
public void inboundBufferUpdated(final ChannelHandlerContext ctx) throws Exception {
final ByteBuf in = ctx.inboundByteBuffer();
if (in.readableBytes() < 5) {
return;
}
int packetLength = getEncryptedPacketLength(in);
if (packetLength == -1) {
// Bad data - discard the buffer and raise an exception.
NotSslRecordException e =
new NotSslRecordException("not an SSL/TLS record: " + ByteBufUtil.hexDump(in));
in.skipBytes(in.readableBytes());
ctx.fireExceptionCaught(e);
setHandshakeFailure(e);
return;
}
assert packetLength > 0;
final ByteBuf out = ctx.nextInboundByteBuffer();
out.discardReadBytes();
boolean wrapLater = false;
int bytesProduced = 0;
try {
loop:
for (; ; ) {
SSLEngineResult result = unwrap(engine, in, out);
bytesProduced += result.bytesProduced();
switch (result.getStatus()) {
case CLOSED:
// notify about the CLOSED state of the SSLEngine. See #137
sslCloseFuture.setClosed();
break;
case BUFFER_UNDERFLOW:
break loop;
}
switch (result.getHandshakeStatus()) {
case NEED_UNWRAP:
break;
case NEED_WRAP:
wrapLater = true;
break;
case NEED_TASK:
runDelegatedTasks();
break;
case FINISHED:
setHandshakeSuccess();
wrapLater = true;
continue;
case NOT_HANDSHAKING:
break;
default:
throw new IllegalStateException(
"Unknown handshake status: " + result.getHandshakeStatus());
}
if (result.bytesConsumed() == 0 && result.bytesProduced() == 0) {
break;
}
}
if (wrapLater) {
flush(ctx, ctx.newFuture());
}
} catch (SSLException e) {
setHandshakeFailure(e);
throw e;
} finally {
if (bytesProduced > 0) {
in.discardReadBytes();
ctx.fireInboundBufferUpdated();
}
}
}
@Override
public void flush(final ChannelHandlerContext ctx, ChannelFuture future) throws Exception {
final ByteBuf in = ctx.outboundByteBuffer();
final ByteBuf out = ctx.nextOutboundByteBuffer();
out.unsafe().discardSomeReadBytes();
// Do not encrypt the first write request if this handler is
// created with startTLS flag turned on.
if (startTls && !sentFirstMessage) {
sentFirstMessage = true;
out.writeBytes(in);
ctx.flush(future);
return;
}
if (ctx.executor() == ctx.channel().eventLoop()) {
flushFutureNotifier.addFlushFuture(future, in.readableBytes());
} else {
synchronized (flushFutureNotifier) {
flushFutureNotifier.addFlushFuture(future, in.readableBytes());
}
}
boolean unwrapLater = false;
int bytesConsumed = 0;
try {
for (; ; ) {
SSLEngineResult result = wrap(engine, in, out);
bytesConsumed += result.bytesConsumed();
if (result.getStatus() == Status.CLOSED) {
// SSLEngine has been closed already.
// Any further write attempts should be denied.
if (in.readable()) {
in.clear();
SSLException e = new SSLException("SSLEngine already closed");
future.setFailure(e);
ctx.fireExceptionCaught(e);
flush0(ctx, bytesConsumed, e);
bytesConsumed = 0;
}
break;
} else {
switch (result.getHandshakeStatus()) {
case NEED_WRAP:
ctx.flush();
continue;
case NEED_UNWRAP:
if (ctx.inboundByteBuffer().readable()) {
unwrapLater = true;
}
break;
case NEED_TASK:
runDelegatedTasks();
continue;
case FINISHED:
setHandshakeSuccess();
continue;
case NOT_HANDSHAKING:
break;
default:
throw new IllegalStateException(
"Unknown handshake status: " + result.getHandshakeStatus());
}
if (result.bytesConsumed() == 0 && result.bytesProduced() == 0) {
break;
}
}
}
if (unwrapLater) {
inboundBufferUpdated(ctx);
}
} catch (SSLException e) {
setHandshakeFailure(e);
throw e;
} finally {
in.unsafe().discardSomeReadBytes();
flush0(ctx, bytesConsumed);
}
}
/** Unwraps a single SSL record. */
private void unwrapSingle(
ChannelHandlerContext ctx, ByteBuffer packet, int initialOutAppBufCapacity)
throws SSLException {
boolean wrapLater = false;
try {
for (; ; ) {
if (decodeOut == null) {
decodeOut = ctx.alloc().buffer(initialOutAppBufCapacity);
}
final SSLEngineResult result = unwrap(engine, packet, decodeOut);
final Status status = result.getStatus();
final HandshakeStatus handshakeStatus = result.getHandshakeStatus();
final int produced = result.bytesProduced();
final int consumed = result.bytesConsumed();
if (status == Status.CLOSED) {
// notify about the CLOSED state of the SSLEngine. See #137
sslCloseFuture.trySuccess(ctx.channel());
break;
}
switch (handshakeStatus) {
case NEED_UNWRAP:
break;
case NEED_WRAP:
wrapNonAppData(ctx, true);
break;
case NEED_TASK:
runDelegatedTasks();
break;
case FINISHED:
setHandshakeSuccess();
wrapLater = true;
continue;
case NOT_HANDSHAKING:
if (setHandshakeSuccessIfStillHandshaking()) {
wrapLater = true;
continue;
}
if (flushedBeforeHandshakeDone) {
// We need to call wrap(...) in case there was a flush done before the handshake
// completed.
//
// See https://github.com/netty/netty/pull/2437
flushedBeforeHandshakeDone = false;
wrapLater = true;
}
break;
default:
throw new IllegalStateException("Unknown handshake status: " + handshakeStatus);
}
if (status == Status.BUFFER_UNDERFLOW || consumed == 0 && produced == 0) {
break;
}
}
if (wrapLater) {
wrap(ctx, true);
}
} catch (SSLException e) {
setHandshakeFailure(e);
throw e;
}
}
private void wrap(ChannelHandlerContext ctx, boolean inUnwrap) throws SSLException {
ByteBuf out = null;
ChannelPromise promise = null;
try {
for (; ; ) {
PendingWrite pending = pendingUnencryptedWrites.peek();
if (pending == null) {
break;
}
if (out == null) {
out = ctx.alloc().buffer(maxPacketBufferSize);
}
if (!(pending.msg() instanceof ByteBuf)) {
ctx.write(pending.msg(), (ChannelPromise) pending.recycleAndGet());
pendingUnencryptedWrites.remove();
continue;
}
ByteBuf buf = (ByteBuf) pending.msg();
SSLEngineResult result = wrap(engine, buf, out);
if (!buf.isReadable()) {
buf.release();
promise = (ChannelPromise) pending.recycleAndGet();
pendingUnencryptedWrites.remove();
} else {
promise = null;
}
if (result.getStatus() == Status.CLOSED) {
// SSLEngine has been closed already.
// Any further write attempts should be denied.
for (; ; ) {
PendingWrite w = pendingUnencryptedWrites.poll();
if (w == null) {
break;
}
w.failAndRecycle(SSLENGINE_CLOSED);
}
return;
} else {
switch (result.getHandshakeStatus()) {
case NEED_TASK:
runDelegatedTasks();
break;
case FINISHED:
setHandshakeSuccess();
// deliberate fall-through
case NOT_HANDSHAKING:
setHandshakeSuccessIfStillHandshaking();
// deliberate fall-through
case NEED_WRAP:
finishWrap(ctx, out, promise, inUnwrap);
promise = null;
out = null;
break;
case NEED_UNWRAP:
return;
default:
throw new IllegalStateException(
"Unknown handshake status: " + result.getHandshakeStatus());
}
}
}
} catch (SSLException e) {
setHandshakeFailure(e);
throw e;
} finally {
finishWrap(ctx, out, promise, inUnwrap);
}
}