/**
* Return a readOnly slice of this buffer. Flips the internal buffer. May not be, usefully,
* invoked multiple times on the same internal state.
*
* <p>Only use this if using a non-direct ByteBuffer!
*/
public ByteBuffer getBuffer() {
assert (isDirect == false);
assert (buffer.b().hasArray());
assert (!buffer.b().isDirect());
buffer.b().flip();
return buffer.b().asReadOnlyBuffer();
}
BBContainer acquire() {
final BBContainer cont = m_buffers.poll();
if (cont == null) {
final BBContainer originContainer = DBBPool.allocateDirect(1024 * 32);
return new BBContainer(originContainer.b()) {
@Override
public void discard() {
checkDoubleFree();
// If we had to allocate over the desired limit, start discarding
if (m_buffers.size() > m_numBuffers) {
originContainer.discard();
return;
}
m_buffers.push(originContainer);
}
};
}
return new BBContainer(cont.b()) {
@Override
public void discard() {
checkDoubleFree();
m_buffers.push(cont);
}
};
}
/**
* Get a ascii-string-safe version of the binary value using a hex encoding.
*
* @return A hex-encoded string value representing the serialized objects.
*/
public String getHexEncodedBytes() {
buffer.b().flip();
byte bytes[] = new byte[buffer.b().remaining()];
buffer.b().get(bytes);
String hex = Encoder.hexEncode(bytes);
buffer.discard();
return hex;
}
/**
* This method is slow and horrible. It entails an extra copy. Don't use it! Ever! Not even for
* test! Just say no to test only code. It will also leak the BBContainer if this FS is being used
* with a pool.
*/
public byte[] getBytes() {
byte[] retval = new byte[buffer.b().position()];
int position = buffer.b().position();
buffer.b().rewind();
buffer.b().get(retval);
assert position == buffer.b().position();
return retval;
}
public void writeArray(BigDecimal[] values) throws IOException {
if (values.length > Short.MAX_VALUE) {
throw new IOException("Array exceeds maximum length of " + Short.MAX_VALUE + " bytes");
}
writeShort(values.length);
growIfNeeded(16); // sizeof bigdecimal
for (int i = 0; i < values.length; ++i) {
if (values[i] == null) {
VoltDecimalHelper.serializeNull(buffer.b());
} else {
VoltDecimalHelper.serializeBigDecimal(values[i], buffer.b());
}
}
}
/** Resizes the internal byte buffer with a simple doubling policy, if needed. */
private final void growIfNeeded(int minimumDesired) {
if (buffer.b().remaining() < minimumDesired) {
// Compute the size of the new buffer
int newCapacity = buffer.b().capacity();
int newRemaining = newCapacity - buffer.b().position();
while (newRemaining < minimumDesired) {
newRemaining += newCapacity;
newCapacity *= 2;
}
// Allocate and copy
BBContainer next;
if (isDirect) {
next = DBBPool.allocateDirect(newCapacity);
} else {
next = DBBPool.wrapBB(ByteBuffer.allocate(newCapacity));
}
buffer.b().flip();
next.b().put(buffer.b());
assert next.b().remaining() == newRemaining;
buffer.discard();
buffer = next;
if (callback != null) callback.onBufferGrow(this);
assert (buffer.b().order() == ByteOrder.BIG_ENDIAN);
}
}
private void convertChunks() throws IOException, InterruptedException {
int lastNumCharacters = 1024 * 64;
while (!Thread.interrupted() && m_saveFile.hasMoreChunks()) {
if (m_availableBytes.get() > m_maxAvailableBytes) {
Thread.sleep(5);
continue;
}
BBContainer c = m_saveFile.getNextChunk();
if (c == null) {
return;
}
try {
final VoltTable vt = PrivateVoltTableFactory.createVoltTableFromBuffer(c.b(), true);
Pair<Integer, byte[]> p = VoltTableUtil.toCSV(vt, m_delimiter, null, lastNumCharacters);
lastNumCharacters = p.getFirst();
byte csvBytes[] = p.getSecond();
// should not insert empty byte[] if not last ConverterThread
if (csvBytes.length > 0) {
m_availableBytes.addAndGet(csvBytes.length);
m_available.offer(csvBytes);
}
} finally {
c.discard();
}
}
}
/** constructor that sets callback object. */
public FastSerializer(
boolean bigEndian, boolean isDirect, BufferGrowCallback callback, int initialAllocation) {
assert (initialAllocation > 0);
this.isDirect = isDirect;
if (isDirect) {
buffer = DBBPool.allocateDirect(initialAllocation);
} else {
buffer = DBBPool.wrapBB(ByteBuffer.allocate(initialAllocation));
}
this.callback = callback;
buffer.b().order(bigEndian ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN);
}
@Override
public void writeByte(int v) throws IOException {
growIfNeeded(Byte.SIZE / 8);
buffer.b().put((byte) v);
}
@Override
public void write(byte[] b, int off, int len) throws IOException {
growIfNeeded(len);
buffer.b().put(b, off, len);
}
/** return Current position within the underlying buffer, for self-comparison only. */
public int getPosition() {
return buffer.b().position();
}
@Override
public void writeLong(long v) throws IOException {
growIfNeeded(Long.SIZE / 8);
buffer.b().putLong(v);
}
@Override
public void writeFloat(float v) throws IOException {
growIfNeeded(Float.SIZE / 8);
buffer.b().putFloat(v);
}
/** Write a table using it's ByteBuffer serialization code. */
public void writeTable(VoltTable table) throws IOException {
int len = table.getSerializedSize();
growIfNeeded(len);
table.flattenToBuffer(buffer.b());
}
BBContainer unreleasedContainer() {
m_refCount.incrementAndGet();
return getRefCountingContainer(m_buffer.b().slice().asReadOnlyBuffer());
}
/**
* When a fast serializer is shared between Java and native this is called to retrieve a reference
* to the to buffer without flipping it. OnBufferGrowCallback needs this to update the pointer to
* the shared buffer when the parameter buffer grows.
*/
public BBContainer getContainerNoFlip() {
assert (isDirect == true);
assert (buffer.b().isDirect());
return buffer;
}
/*
* Does not increment the refcount, uses the implicit 1 count
* and should only be called once to get a container for pushing the data to disk
*/
BBContainer asBBContainer() {
m_buffer.b().putLong(0, uso());
m_buffer.b().position(0);
return getRefCountingContainer(m_buffer.b().asReadOnlyBuffer());
}
public int size() {
return buffer.b().position();
}
/** @return a reference to the underlying ByteBuffer. */
public BBContainer getBBContainer() {
buffer.b().flip();
return buffer;
}
@Override
public void writeChar(int v) throws IOException {
growIfNeeded(Character.SIZE / 8);
buffer.b().putChar((char) v);
}
@Override
public void writeDouble(double v) throws IOException {
growIfNeeded(Double.SIZE / 8);
buffer.b().putDouble(v);
}
/** Write an SPI using it's ByteBuffer serialization code. */
public void writeInvocation(StoredProcedureInvocation invocation) throws IOException {
int len = invocation.getSerializedSize();
growIfNeeded(len);
invocation.flattenToBuffer(buffer.b());
}
@Override
public void writeInt(int v) throws IOException {
growIfNeeded(Integer.SIZE / 8);
buffer.b().putInt(v);
}
/** Write a ParameterSet using it's ByteBuffer serialization code. */
public void writeParameterSet(ParameterSet params) throws IOException {
int len = params.getSerializedSize();
growIfNeeded(len);
params.flattenToBuffer(buffer.b());
}
@Override
public void writeShort(int v) throws IOException {
growIfNeeded(Short.SIZE / 8);
buffer.b().putShort((short) v);
}
/**
* Process a message pulled off from the network thread, and discard the container once it's
* processed.
*
* @param msg A pair of <sourceHSId, blockContainer>
* @return The restore work, or null if there's no data block to return to the site.
*/
private RestoreWork processMessage(
Pair<Long, Pair<Long, BBContainer>> msg, CachedByteBufferAllocator resultBufferAllocator) {
if (msg == null) {
return null;
}
RestoreWork restoreWork = null;
long hsId = msg.getFirst();
long targetId = msg.getSecond().getFirst();
BBContainer container = msg.getSecond().getSecond();
try {
ByteBuffer block = container.b();
byte typeByte = block.get(StreamSnapshotDataTarget.typeOffset);
final int blockIndex = block.getInt(StreamSnapshotDataTarget.blockIndexOffset);
StreamSnapshotMessageType type = StreamSnapshotMessageType.values()[typeByte];
if (type == StreamSnapshotMessageType.FAILURE) {
VoltDB.crashLocalVoltDB("Rejoin source sent failure message.", false, null);
// for test code only
if (m_expectedEOFs.decrementAndGet() == 0) {
m_EOF = true;
}
} else if (type == StreamSnapshotMessageType.END) {
if (rejoinLog.isTraceEnabled()) {
rejoinLog.trace("Got END message " + blockIndex);
}
// End of stream, no need to ack this buffer
if (m_expectedEOFs.decrementAndGet() == 0) {
m_EOF = true;
}
} else if (type == StreamSnapshotMessageType.SCHEMA) {
rejoinLog.trace("Got SCHEMA message");
block.position(StreamSnapshotDataTarget.contentOffset);
byte[] schemaBytes = new byte[block.remaining()];
block.get(schemaBytes);
m_schemas.put(block.getInt(StreamSnapshotDataTarget.tableIdOffset), schemaBytes);
} else if (type == StreamSnapshotMessageType.HASHINATOR) {
block.position(StreamSnapshotDataTarget.contentOffset);
long version = block.getLong();
byte[] hashinatorConfig = new byte[block.remaining()];
block.get(hashinatorConfig);
restoreWork = new HashinatorRestoreWork(version, hashinatorConfig);
} else {
// It's normal snapshot data afterwards
final int tableId = block.getInt(StreamSnapshotDataTarget.tableIdOffset);
if (!m_schemas.containsKey(tableId)) {
VoltDB.crashLocalVoltDB("No schema for table with ID " + tableId, false, null);
}
// Get the byte buffer ready to be consumed
block.position(StreamSnapshotDataTarget.contentOffset);
ByteBuffer nextChunk = getNextChunk(m_schemas.get(tableId), block, resultBufferAllocator);
m_bytesReceived += nextChunk.remaining();
restoreWork = new TableRestoreWork(tableId, nextChunk);
}
// Queue ack to this block
m_ack.ack(hsId, m_EOF, targetId, blockIndex);
return restoreWork;
} finally {
container.discard();
}
}
/**
* Set current position of underlying buffer. Useful only in concert with getPosition()
*
* @param pos The position to set to.
*/
public void setPosition(int pos) {
buffer.b().position(pos);
}
@Override
public void write(byte[] b) throws IOException {
growIfNeeded(b.length);
buffer.b().put(b);
}
public void write(ByteBuffer b) throws IOException {
growIfNeeded(b.limit() - b.position());
buffer.b().put(b);
}
/** Clears the contents of the underlying buffer, making iteady for more writes. */
public void clear() {
buffer.b().clear();
}