@Test
public void containsKeyColumnReturnsFalseOnNonexistentInput() throws Exception {
TransactionHandle txn = manager.beginTransaction();
ByteBuffer key1 = KeyColumnValueStoreUtil.longToByteBuffer(1);
ByteBuffer c = KeyColumnValueStoreUtil.stringToByteBuffer("c");
assertFalse(store.containsKeyColumn(key1.duplicate(), c.duplicate(), txn));
txn.commit();
}
/** Helper method for implementing {@link MessageLite#equals()} for bytes field. */
public static boolean equalsByteBuffer(ByteBuffer a, ByteBuffer b) {
if (a.capacity() != b.capacity()) {
return false;
}
// ByteBuffer.equals() will only compare the remaining bytes, but we want to
// compare all the content.
return a.duplicate().clear().equals(b.duplicate().clear());
}
@SuppressWarnings("unchecked")
public ByteBuffer serialize() {
if (serialized != null) {
return serialized.duplicate();
}
ByteBufferOutputStream out = new ByteBufferOutputStream();
int i = 0;
for (Component c : components) {
Serializer<?> s = serializerForPosition(i);
ByteBuffer cb = c.getBytes(s);
if (cb == null) {
cb = ByteBuffer.allocate(0);
}
if (dynamic) {
String comparator = comparatorForPosition(i);
if (comparator == null) {
comparator = c.getComparator();
}
if (comparator == null) {
comparator = BYTESTYPE.getTypeName();
}
int p = comparator.indexOf("(reversed=true)");
boolean desc = false;
if (p >= 0) {
comparator = comparator.substring(0, p);
desc = true;
}
if (aliasToComparatorMapping.inverse().containsKey(comparator)) {
byte a = aliasToComparatorMapping.inverse().get(comparator);
if (desc) {
a = (byte) Character.toUpperCase((char) a);
}
out.writeShort((short) (0x8000 | a));
} else {
out.writeShort((short) comparator.length());
out.write(comparator.getBytes(Charsets.UTF_8));
}
// if (comparator.equals(BYTESTYPE.getTypeName()) && (cb.remaining() ==
// 0)) {
// log.warn("Writing zero-length BytesType, probably an error");
// }
}
out.writeShort((short) cb.remaining());
out.write(cb.slice());
if ((i == (components.size() - 1)) && (equality != ComponentEquality.EQUAL)) {
out.write(equality.toByte());
} else {
out.write(c.getEquality().toByte());
}
i++;
}
serialized = out.getByteBuffer();
return serialized.duplicate();
}
@Test
public void containsKeyColumnReturnsTrueOnExtantInput() throws Exception {
TransactionHandle txn = manager.beginTransaction();
KeyColumnValueStoreUtil.insert(store, txn, 1, "c", "v");
txn.commit();
txn = manager.beginTransaction();
ByteBuffer key1 = KeyColumnValueStoreUtil.longToByteBuffer(1);
ByteBuffer c = KeyColumnValueStoreUtil.stringToByteBuffer("c");
assertTrue(store.containsKeyColumn(key1.duplicate(), c.duplicate(), txn));
txn.commit();
}
private final SendBuffer acquire(SctpPayload message) {
final ChannelBuffer src = message.getPayloadBuffer();
final int streamNo = message.getStreamIdentifier();
final int protocolId = message.getProtocolIdentifier();
final int size = src.readableBytes();
if (size == 0) {
return EMPTY_BUFFER;
}
if (src.isDirect()) {
return new UnpooledSendBuffer(streamNo, protocolId, src.toByteBuffer());
}
if (src.readableBytes() > DEFAULT_PREALLOCATION_SIZE) {
return new UnpooledSendBuffer(streamNo, protocolId, src.toByteBuffer());
}
Preallocation current = this.current;
ByteBuffer buffer = current.buffer;
int remaining = buffer.remaining();
PooledSendBuffer dst;
if (size < remaining) {
int nextPos = buffer.position() + size;
ByteBuffer slice = buffer.duplicate();
buffer.position(align(nextPos));
slice.limit(nextPos);
current.refCnt++;
dst = new PooledSendBuffer(streamNo, protocolId, current, slice);
} else if (size > remaining) {
this.current = current = getPreallocation();
buffer = current.buffer;
ByteBuffer slice = buffer.duplicate();
buffer.position(align(size));
slice.limit(size);
current.refCnt++;
dst = new PooledSendBuffer(streamNo, protocolId, current, slice);
} else { // size == remaining
current.refCnt++;
this.current = getPreallocation0();
dst = new PooledSendBuffer(streamNo, protocolId, current, current.buffer);
}
ByteBuffer dstbuf = dst.buffer;
dstbuf.mark();
src.getBytes(src.readerIndex(), dstbuf);
dstbuf.reset();
return dst;
}
private SendBuffer acquire(ChannelBuffer src) {
final int size = src.readableBytes();
if (size == 0) {
return EMPTY_BUFFER;
}
if (src instanceof CompositeChannelBuffer && ((CompositeChannelBuffer) src).useGathering()) {
return new GatheringSendBuffer(src.toByteBuffers());
}
if (src.isDirect()) {
return new UnpooledSendBuffer(src.toByteBuffer());
}
if (src.readableBytes() > DEFAULT_PREALLOCATION_SIZE) {
return new UnpooledSendBuffer(src.toByteBuffer());
}
Preallocation current = this.current;
ByteBuffer buffer = current.buffer;
int remaining = buffer.remaining();
PooledSendBuffer dst;
if (size < remaining) {
int nextPos = buffer.position() + size;
ByteBuffer slice = buffer.duplicate();
buffer.position(align(nextPos));
slice.limit(nextPos);
current.refCnt++;
dst = new PooledSendBuffer(current, slice);
} else if (size > remaining) {
this.current = current = getPreallocation();
buffer = current.buffer;
ByteBuffer slice = buffer.duplicate();
buffer.position(align(size));
slice.limit(size);
current.refCnt++;
dst = new PooledSendBuffer(current, slice);
} else { // size == remaining
current.refCnt++;
this.current = getPreallocation0();
dst = new PooledSendBuffer(current, current.buffer);
}
ByteBuffer dstbuf = dst.buffer;
dstbuf.mark();
src.getBytes(src.readerIndex(), dstbuf);
dstbuf.reset();
return dst;
}
@Override
public int write(ByteBuffer src) throws IOException {
ByteBuffer dup = src.duplicate();
int res = this.iwc.write(src);
this.messageDigest.update(dup);
return res;
}
private ByteBuffer internalNioBuffer() {
ByteBuffer tmpNioBuf = this.tmpNioBuf;
if (tmpNioBuf == null) {
this.tmpNioBuf = tmpNioBuf = buffer.duplicate();
}
return tmpNioBuf;
}
@Override
protected void onFullCloseMessage(
final WebSocketChannel channel, final BufferedBinaryMessage message) {
final Pooled<ByteBuffer[]> pooled = message.getData();
final ByteBuffer singleBuffer = toBuffer(pooled.getResource());
final ByteBuffer toSend = singleBuffer.duplicate();
session
.getContainer()
.invokeEndpointMethod(
executor,
new Runnable() {
@Override
public void run() {
WebSockets.sendClose(toSend, channel, null);
try {
if (singleBuffer.remaining() > 2) {
final int code = singleBuffer.getShort();
session.close(
new javax.websocket.CloseReason(
javax.websocket.CloseReason.CloseCodes.getCloseCode(code),
new UTF8Output(singleBuffer).extract()));
} else {
session.close();
}
} catch (IOException e) {
invokeOnError(e);
} finally {
pooled.free();
}
}
});
}
public void serve(int port) throws IOException {
ServerSocketChannel serverChannel = ServerSocketChannel.open();
serverChannel.configureBlocking(false);
ServerSocket ss = serverChannel.socket();
InetSocketAddress address = new InetSocketAddress(port);
// Binds the server to the selected port
ss.bind(address);
// Opens the Selector for handling channels
Selector selector = Selector.open();
// Registers the ServerSocket with the Selector to accept connections
serverChannel.register(selector, SelectionKey.OP_ACCEPT);
final ByteBuffer msg = ByteBuffer.wrap("Hi!\r\n".getBytes());
while (true) {
try {
// Waits for new events to process; blocks until the next incoming event
selector.select();
} catch (IOException e) {
e.printStackTrace();
break;
}
// Obtains all SelectionKey instances that received events
Set<SelectionKey> readyKeys = selector.selectedKeys();
Iterator<SelectionKey> iterator = readyKeys.iterator();
while (iterator.hasNext()) {
SelectionKey key = iterator.next();
iterator.remove();
try {
// Checks if the event is a new connection ready to be accepted
if (key.isAcceptable()) {
ServerSocketChannel server = (ServerSocketChannel) key.channel();
SocketChannel client = server.accept();
client.configureBlocking(false);
// Accepts client and registers it with the selector
client.register(
selector, SelectionKey.OP_WRITE | SelectionKey.OP_READ, msg.duplicate());
System.out.println("Accepted connection from " + client);
}
// Checks if the socket is ready for writing data
if (key.isWritable()) {
SocketChannel client = (SocketChannel) key.channel();
ByteBuffer buffer = (ByteBuffer) key.attachment();
while (buffer.hasRemaining()) {
// Writes data to the connected client
if (client.write(buffer) == 0) {
break;
}
}
// Closes the connection
client.close();
}
} catch (IOException e) {
key.cancel();
try {
key.channel().close();
} catch (IOException e1) {
}
}
}
}
}
/** Helper method for implementing {@link MessageLite#hashCode()} for bytes field. */
public static int hashCodeByteBuffer(ByteBuffer bytes) {
if (bytes.hasArray()) {
// Fast path.
int h =
LiteralByteString.hashCode(
bytes.capacity(), bytes.array(), bytes.arrayOffset(), bytes.capacity());
return h == 0 ? 1 : h;
} else {
// Read the data into a temporary byte array before calculating the
// hash value.
final int bufferSize =
bytes.capacity() > DEFAULT_BUFFER_SIZE ? DEFAULT_BUFFER_SIZE : bytes.capacity();
final byte[] buffer = new byte[bufferSize];
final ByteBuffer duplicated = bytes.duplicate();
duplicated.clear();
int h = bytes.capacity();
while (duplicated.remaining() > 0) {
final int length =
duplicated.remaining() <= bufferSize ? duplicated.remaining() : bufferSize;
duplicated.get(buffer, 0, length);
h = LiteralByteString.hashCode(h, buffer, 0, length);
}
return h == 0 ? 1 : h;
}
}
public AsyncWrite(byte[] b, int off, int len, boolean complete) {
_buffer = ByteBuffer.wrap(b, off, len);
_len = len;
// always use a view for large byte arrays to avoid JVM pooling large direct buffers
_slice = _len < getBufferSize() ? null : _buffer.duplicate();
_complete = complete;
}
public synchronized int send(short channel, FrameBody body, ByteBuffer payload) {
if (!_closedForOutput) {
ValueWriter<FrameBody> writer = _describedTypeRegistry.getValueWriter(body);
int size = writer.writeToBuffer(EMPTY_BYTE_BUFFER);
ByteBuffer payloadDup = payload == null ? null : payload.duplicate();
int payloadSent = getMaxFrameSize() - (size + 9);
if (payloadSent < (payload == null ? 0 : payload.remaining())) {
if (body instanceof Transfer) {
((Transfer) body).setMore(Boolean.TRUE);
}
writer = _describedTypeRegistry.getValueWriter(body);
size = writer.writeToBuffer(EMPTY_BYTE_BUFFER);
payloadSent = getMaxFrameSize() - (size + 9);
try {
payloadDup.limit(payloadDup.position() + payloadSent);
} catch (NullPointerException npe) {
throw npe;
}
} else {
payloadSent = payload == null ? 0 : payload.remaining();
}
_frameOutputHandler.send(AMQFrame.createAMQFrame(channel, body, payloadDup));
return payloadSent;
} else {
return -1;
}
}
@Override
public void run() {
threadCnt.incrementAndGet();
try {
while (running && !shutdown && sock != null) {
/** Reads the first int to determine the length of the message */
int length = din.readInt();
if (length <= 0 || length > PACKETMAXSIZE) {
throw new IOException("Received packet with invalid packet: " + length);
}
/** Allocates a new ByteBuffer to receive the message */
byte[] msgArray = new byte[length];
din.readFully(msgArray, 0, length);
ByteBuffer message = ByteBuffer.wrap(msgArray);
addToRecvQueue(new Message(message.duplicate(), sid));
}
} catch (Exception e) {
LOG.warn("Connection broken for id " + sid + ", my id = " + self.getId() + ", error = ", e);
} finally {
LOG.warn("Interrupting SendWorker");
sw.finish();
if (sock != null) {
closeSocket(sock);
}
}
}
/**
* read from a file channel into a byte buffer, starting at a certain position.
*
* @param channel channel to read from
* @param channelPosition position to read from
* @param dest destination {@link java.nio.ByteBuffer} to put data in
* @return total bytes read or -1 if an attempt was made to read past EOF. The method always tries
* to read all the bytes that will fit in the destination byte buffer.
*/
public static int readFromFileChannel(FileChannel channel, long channelPosition, ByteBuffer dest)
throws IOException {
if (dest.isDirect() || (dest.remaining() < READ_CHUNK_SIZE)) {
return readSingleChunk(channel, channelPosition, dest);
} else {
int bytesRead = 0;
int bytesToRead = dest.remaining();
// duplicate the buffer in order to be able to change the limit
ByteBuffer tmpBuffer = dest.duplicate();
try {
while (dest.hasRemaining()) {
tmpBuffer.limit(Math.min(dest.limit(), tmpBuffer.position() + READ_CHUNK_SIZE));
int read = readSingleChunk(channel, channelPosition, tmpBuffer);
if (read < 0) {
return read;
}
bytesRead += read;
channelPosition += read;
dest.position(tmpBuffer.position());
}
} finally {
// make sure we update byteBuffer to indicate how far we came..
dest.position(tmpBuffer.position());
}
assert bytesRead == bytesToRead
: "failed to read an entire buffer but also didn't get an EOF (read ["
+ bytesRead
+ "] needed ["
+ bytesToRead
+ "]";
return bytesRead;
}
}
/**
* Read as many bytes into dest as dest.capacity() starting at position pos in the FileChannel.
* This function can read from the buffer or the file channel depending on the implementation..
*
* @param dest
* @param pos
* @return The total number of bytes read.
* @throws IOException if a read operation fails or in case of a short read.
*/
public synchronized int read(ByteBuffer dest, long pos) throws IOException {
invocationCount++;
long currentPosition = pos;
while (dest.remaining() > 0) {
// Check if the data is in the buffer, if so, copy it.
if (readBufferStartPosition <= currentPosition
&& currentPosition < readBufferStartPosition + readBuffer.limit()) {
long posInBuffer = currentPosition - readBufferStartPosition;
long bytesToCopy = Math.min(dest.remaining(), readBuffer.limit() - posInBuffer);
ByteBuffer rbDup = readBuffer.duplicate();
rbDup.position((int) posInBuffer);
rbDup.limit((int) (posInBuffer + bytesToCopy));
dest.put(rbDup);
currentPosition += bytesToCopy;
cacheHitCount++;
} else {
// We don't have it in the buffer, so put necessary data in the buffer
readBuffer.clear();
readBufferStartPosition = currentPosition;
int readBytes = 0;
if ((readBytes = validateAndGetFileChannel().read(readBuffer, currentPosition)) <= 0) {
throw new IOException(
"Reading from filechannel returned a non-positive value. Short read.");
}
readBuffer.limit(readBytes);
}
}
return (int) (currentPosition - pos);
}
@Override
public BytesReference slice(int from, int length) {
ByteBuffer dup = buffer.duplicate();
dup.position(buffer.position() + from);
dup.limit(buffer.position() + from + length);
return new ByteBufferBytesReference(dup);
}
/**
* Processes invoke this message to queue a message to send. Currently, only leader election uses
* it.
*/
public void toSend(Long sid, ByteBuffer b) {
/*
* If sending message to myself, then simply enqueue it (loopback).
*/
if (self.getId() == sid) {
b.position(0);
addToRecvQueue(new Message(b.duplicate(), sid));
/*
* Otherwise send to the corresponding thread to send.
*/
} else {
/*
* Start a new connection if doesn't have one already.
*/
if (!queueSendMap.containsKey(sid)) {
ArrayBlockingQueue<ByteBuffer> bq = new ArrayBlockingQueue<ByteBuffer>(SEND_CAPACITY);
queueSendMap.put(sid, bq);
addToSendQueue(bq, b);
} else {
ArrayBlockingQueue<ByteBuffer> bq = queueSendMap.get(sid);
if (bq != null) {
addToSendQueue(bq, b);
} else {
LOG.error("No queue for server " + sid);
}
}
connectOne(sid);
}
}
@Override
public boolean tryAdvance(Consumer<? super ByteBuffer> action) {
Objects.requireNonNull(action);
int messageOffset = offset + MessageLengthFrameDecoder.HEADER_LENGTH;
if (messageOffset > buffer.limit()) {
return false;
}
frameDecoder.wrap(buffer);
frameDecoder.decodeFrameHeader();
final int messageLength = frameDecoder.getMessageLength();
int messageLimit = offset + MessageLengthFrameDecoder.HEADER_LENGTH + messageLength;
if (messageLength <= 0 || (messageLimit > buffer.limit())) {
return false;
}
ByteBuffer message = buffer.duplicate();
message.order(originalByteOrder);
message.limit(messageLimit);
action.accept(message);
offset = messageLimit;
buffer.position(messageLimit);
return true;
}
public static void print(PrintStream out, ByteBuffer byteBuffer) {
byteBuffer = byteBuffer.duplicate();
out.printf(
"ByteBuffer[pos=%d lim=%d cap=%d]",
byteBuffer.position(), byteBuffer.limit(), byteBuffer.capacity());
// Print bytes.
int i = 0;
int maxBytes = 3 * 20;
for (; i < maxBytes && i < byteBuffer.limit(); i++) {
if (i % 20 == 0) out.println();
out.printf(" 0x%02X", byteBuffer.get(i)); // Does not move position
}
if (i < byteBuffer.limit()) {
if (i % 24 == 0) out.println();
out.print(" ...");
}
// Print as 4-byte ints
// int maxSlots = 8 ;
// int i = 0 ;
// for ( ; i < maxSlots && 4*i < byteBuffer.limit() ; i++ )
// out.printf(" 0x%04X", byteBuffer.getInt(4*i)) ;
// if ( i < maxSlots )
// out.print(" ...") ;
out.println();
}
public static String bytes2HexStr(ByteBuffer paramByteBuffer) {
paramByteBuffer = paramByteBuffer.duplicate();
paramByteBuffer.flip();
byte[] arrayOfByte = new byte[paramByteBuffer.limit()];
paramByteBuffer.get(arrayOfByte);
return bytes2HexStr(arrayOfByte);
}
/**
* Copies bytes from the channel buffer into a destination <tt>ByteBuffer</tt>
*
* @param dst A <tt>ByteBuffer</tt> to place the data in.
* @return The number of bytes copied.
*/
private final int copyBufferedBytes(ByteBuffer dst) {
final int bytesToCopy = dst.remaining();
if (ungotc != -1 && dst.hasRemaining()) {
dst.put((byte) ungotc);
ungotc = -1;
}
if (buffer.hasRemaining() && dst.hasRemaining()) {
if (dst.remaining() >= buffer.remaining()) {
//
// Copy out any buffered bytes
//
dst.put(buffer);
} else {
//
// Need to clamp source (buffer) size to avoid overrun
//
ByteBuffer tmp = buffer.duplicate();
tmp.limit(tmp.position() + dst.remaining());
dst.put(tmp);
buffer.position(tmp.position());
}
}
return bytesToCopy - dst.remaining();
}
@Override
protected ByteBuffer readPes(
SeekableByteChannel ch, long pesPosition, int pesSize, int payloadSize, int pesAbsIdx)
throws IOException {
ch.position(pesPosition * 188);
ByteBuffer buf = NIOUtils.fetchFrom(ch, pesSize * 188);
// NOW REMOVE THE TS CRAP
ByteBuffer dst = buf.duplicate();
while (buf.hasRemaining()) {
ByteBuffer tsBuf = NIOUtils.read(buf, 188);
Assert.assertEquals(0x47, tsBuf.get() & 0xff);
int guidFlags = ((tsBuf.get() & 0xff) << 8) | (tsBuf.get() & 0xff);
int guid = (int) guidFlags & 0x1fff;
if (guid == targetGuid) {
int b0 = tsBuf.get() & 0xff;
int counter = b0 & 0xf;
if ((b0 & 0x20) != 0) {
NIOUtils.skip(tsBuf, tsBuf.get() & 0xff);
}
dst.put(tsBuf);
}
}
dst.flip();
readPESHeader(dst, 0);
dst.limit(dst.position() + payloadSize);
return dst;
}
@Override
public List<Framedata> translateFrame(ByteBuffer buffer)
throws LimitExedeedException, InvalidDataException {
List<Framedata> frames = new LinkedList<Framedata>();
Framedata cur;
if (incompleteframe != null) {
// complete an incomplete frame
while (true) {
try {
buffer.mark();
int available_next_byte_count = buffer.remaining(); // The number of bytes received
int expected_next_byte_count =
incompleteframe.remaining(); // The number of bytes to complete the incomplete frame
if (expected_next_byte_count > available_next_byte_count) {
// did not receive enough bytes to complete the frame
incompleteframe.put(buffer.array(), buffer.position(), available_next_byte_count);
buffer.position(buffer.position() + available_next_byte_count);
return Collections.emptyList();
}
incompleteframe.put(buffer.array(), buffer.position(), expected_next_byte_count);
buffer.position(buffer.position() + expected_next_byte_count);
cur = translateSingleFrame((ByteBuffer) incompleteframe.duplicate().position(0));
frames.add(cur);
incompleteframe = null;
break; // go on with the normal frame receival
} catch (IncompleteException e) {
// extending as much as suggested
@SuppressWarnings("unused")
int oldsize = incompleteframe.limit();
ByteBuffer extendedframe = ByteBuffer.allocate(checkAlloc(e.getPreferedSize()));
assert (extendedframe.limit() > incompleteframe.limit());
incompleteframe.rewind();
extendedframe.put(incompleteframe);
incompleteframe = extendedframe;
return translateFrame(buffer);
}
}
}
while (buffer.hasRemaining()) { // Read as much as possible full frames
buffer.mark();
try {
cur = translateSingleFrame(buffer);
frames.add(cur);
} catch (IncompleteException e) {
// remember the incomplete data
buffer.reset();
int pref = e.getPreferedSize();
incompleteframe = ByteBuffer.allocate(checkAlloc(pref));
incompleteframe.put(buffer);
break;
}
}
return frames;
}
@Override
public ByteBuffer serialize() {
ByteBuffer out = ByteBuffer.allocate(getSize());
IsoTypeWriter.writeUInt8(out, tag);
writeSize(out, getContentSize());
out.put(data.duplicate());
return out;
}
public void addPps(List<ByteBuffer> ppsList) {
for (ByteBuffer byteBuffer : ppsList) {
ByteBuffer dup = byteBuffer.duplicate();
unescapeNAL(dup);
PictureParameterSet p = PictureParameterSet.read(dup);
pps.put(p.pic_parameter_set_id, p);
}
}
public ByteBuffer getBytesUnsafe(int i) {
metadata.checkBounds(i);
ByteBuffer value = data.get(i);
if (value == null) return null;
return value.duplicate();
}
// write a tuple (counter id, clock, count) at an absolute (bytebuffer-wise) offset
private void writeElementAtOffset(
ByteBuffer ctx, int offset, CounterId id, long clock, long count) {
ctx = ctx.duplicate();
ctx.position(offset);
ctx.put(id.bytes().duplicate());
ctx.putLong(clock);
ctx.putLong(count);
}
/**
* get the real data without message header
*
* @return message data(without header)
*/
public ByteBuffer payload() {
ByteBuffer payload = buffer.duplicate();
payload.position(headerSize(magic()));
payload = payload.slice();
payload.limit(payloadSize());
payload.rewind();
return payload;
}
public void addSps(List<ByteBuffer> spsList) {
for (ByteBuffer byteBuffer : spsList) {
ByteBuffer dup = byteBuffer.duplicate();
unescapeNAL(dup);
SeqParameterSet s = SeqParameterSet.read(dup);
sps.put(s.seq_parameter_set_id, s);
}
}