private void convertChunks() throws IOException, InterruptedException {
int lastNumCharacters = 1024 * 64;
while (!Thread.interrupted() && m_saveFile.hasMoreChunks()) {
if (m_availableBytes.get() > m_maxAvailableBytes) {
Thread.sleep(5);
continue;
}
BBContainer c = m_saveFile.getNextChunk();
if (c == null) {
return;
}
try {
final VoltTable vt = PrivateVoltTableFactory.createVoltTableFromBuffer(c.b(), true);
Pair<Integer, byte[]> p = VoltTableUtil.toCSV(vt, m_delimiter, null, lastNumCharacters);
lastNumCharacters = p.getFirst();
byte csvBytes[] = p.getSecond();
// should not insert empty byte[] if not last ConverterThread
if (csvBytes.length > 0) {
m_availableBytes.addAndGet(csvBytes.length);
m_available.offer(csvBytes);
}
} finally {
c.discard();
}
}
}
BBContainer acquire() {
final BBContainer cont = m_buffers.poll();
if (cont == null) {
final BBContainer originContainer = DBBPool.allocateDirect(1024 * 32);
return new BBContainer(originContainer.b()) {
@Override
public void discard() {
checkDoubleFree();
// If we had to allocate over the desired limit, start discarding
if (m_buffers.size() > m_numBuffers) {
originContainer.discard();
return;
}
m_buffers.push(originContainer);
}
};
}
return new BBContainer(cont.b()) {
@Override
public void discard() {
checkDoubleFree();
m_buffers.push(cont);
}
};
}
/** Resizes the internal byte buffer with a simple doubling policy, if needed. */
private final void growIfNeeded(int minimumDesired) {
if (buffer.b().remaining() < minimumDesired) {
// Compute the size of the new buffer
int newCapacity = buffer.b().capacity();
int newRemaining = newCapacity - buffer.b().position();
while (newRemaining < minimumDesired) {
newRemaining += newCapacity;
newCapacity *= 2;
}
// Allocate and copy
BBContainer next;
if (isDirect) {
next = DBBPool.allocateDirect(newCapacity);
} else {
next = DBBPool.wrapBB(ByteBuffer.allocate(newCapacity));
}
buffer.b().flip();
next.b().put(buffer.b());
assert next.b().remaining() == newRemaining;
buffer.discard();
buffer = next;
if (callback != null) callback.onBufferGrow(this);
assert (buffer.b().order() == ByteOrder.BIG_ENDIAN);
}
}
/**
* Return a readOnly slice of this buffer. Flips the internal buffer. May not be, usefully,
* invoked multiple times on the same internal state.
*
* <p>Only use this if using a non-direct ByteBuffer!
*/
public ByteBuffer getBuffer() {
assert (isDirect == false);
assert (buffer.b().hasArray());
assert (!buffer.b().isDirect());
buffer.b().flip();
return buffer.b().asReadOnlyBuffer();
}
/**
* Get a ascii-string-safe version of the binary value using a hex encoding.
*
* @return A hex-encoded string value representing the serialized objects.
*/
public String getHexEncodedBytes() {
buffer.b().flip();
byte bytes[] = new byte[buffer.b().remaining()];
buffer.b().get(bytes);
String hex = Encoder.hexEncode(bytes);
buffer.discard();
return hex;
}
/**
* This method is slow and horrible. It entails an extra copy. Don't use it! Ever! Not even for
* test! Just say no to test only code. It will also leak the BBContainer if this FS is being used
* with a pool.
*/
public byte[] getBytes() {
byte[] retval = new byte[buffer.b().position()];
int position = buffer.b().position();
buffer.b().rewind();
buffer.b().get(retval);
assert position == buffer.b().position();
return retval;
}
@Override
public ListenableFuture<?> write(Callable<BBContainer> tupleData, int tableId) {
try {
BBContainer container = tupleData.call();
if (container != null) {
container.discard();
}
} catch (Exception e) {
}
return null;
}
public void writeArray(BigDecimal[] values) throws IOException {
if (values.length > Short.MAX_VALUE) {
throw new IOException("Array exceeds maximum length of " + Short.MAX_VALUE + " bytes");
}
writeShort(values.length);
growIfNeeded(16); // sizeof bigdecimal
for (int i = 0; i < values.length; ++i) {
if (values[i] == null) {
VoltDecimalHelper.serializeNull(buffer.b());
} else {
VoltDecimalHelper.serializeBigDecimal(values[i], buffer.b());
}
}
}
@Override
public void discard() {
if (m_hasMoreChunks == false) {
m_origin.discard();
} else {
m_buffers.add(this);
}
}
/** constructor that sets callback object. */
public FastSerializer(
boolean bigEndian, boolean isDirect, BufferGrowCallback callback, int initialAllocation) {
assert (initialAllocation > 0);
this.isDirect = isDirect;
if (isDirect) {
buffer = DBBPool.allocateDirect(initialAllocation);
} else {
buffer = DBBPool.wrapBB(ByteBuffer.allocate(initialAllocation));
}
this.callback = callback;
buffer.b().order(bigEndian ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN);
}
public void close() throws IOException {
if (m_chunkReaderThread != null) {
m_chunkReaderThread.interrupt();
try {
m_chunkReaderThread.join();
} catch (InterruptedException e) {
throw new IOException(e);
}
}
synchronized (this) {
while (!m_availableChunks.isEmpty()) {
m_availableChunks.poll().discard();
}
notifyAll();
}
/*
* Free buffers used to pull snapshot data in process
*/
BBContainer cont;
while ((cont = m_buffers.poll()) != null) {
cont.discard();
}
}
/**
* Set current position of underlying buffer. Useful only in concert with getPosition()
*
* @param pos The position to set to.
*/
public void setPosition(int pos) {
buffer.b().position(pos);
}
public void discard() {
buffer.discard();
}
@Override
public void writeShort(int v) throws IOException {
growIfNeeded(Short.SIZE / 8);
buffer.b().putShort((short) v);
}
/** return Current position within the underlying buffer, for self-comparison only. */
public int getPosition() {
return buffer.b().position();
}
@Override
public void writeInt(int v) throws IOException {
growIfNeeded(Integer.SIZE / 8);
buffer.b().putInt(v);
}
@Override
public void writeLong(long v) throws IOException {
growIfNeeded(Long.SIZE / 8);
buffer.b().putLong(v);
}
@Override
public void writeDouble(double v) throws IOException {
growIfNeeded(Double.SIZE / 8);
buffer.b().putDouble(v);
}
/** Write an SPI using it's ByteBuffer serialization code. */
public void writeInvocation(StoredProcedureInvocation invocation) throws IOException {
int len = invocation.getSerializedSize();
growIfNeeded(len);
invocation.flattenToBuffer(buffer.b());
}
@Override
public void writeByte(int v) throws IOException {
growIfNeeded(Byte.SIZE / 8);
buffer.b().put((byte) v);
}
@Override
public void writeChar(int v) throws IOException {
growIfNeeded(Character.SIZE / 8);
buffer.b().putChar((char) v);
}
/**
* When a fast serializer is shared between Java and native this is called to retrieve a reference
* to the to buffer without flipping it. OnBufferGrowCallback needs this to update the pointer to
* the shared buffer when the parameter buffer grows.
*/
public BBContainer getContainerNoFlip() {
assert (isDirect == true);
assert (buffer.b().isDirect());
return buffer;
}
/** Write a table using it's ByteBuffer serialization code. */
public void writeTable(VoltTable table) throws IOException {
int len = table.getSerializedSize();
growIfNeeded(len);
table.flattenToBuffer(buffer.b());
}
@Override
public void write(byte[] b) throws IOException {
growIfNeeded(b.length);
buffer.b().put(b);
}
/** @return a reference to the underlying ByteBuffer. */
public BBContainer getBBContainer() {
buffer.b().flip();
return buffer;
}
@Override
public void writeFloat(float v) throws IOException {
growIfNeeded(Float.SIZE / 8);
buffer.b().putFloat(v);
}
public void write(ByteBuffer b) throws IOException {
growIfNeeded(b.limit() - b.position());
buffer.b().put(b);
}
/** Write a ParameterSet using it's ByteBuffer serialization code. */
public void writeParameterSet(ParameterSet params) throws IOException {
int len = params.getSerializedSize();
growIfNeeded(len);
params.flattenToBuffer(buffer.b());
}
public static synchronized void pushDRBuffer(int partitionId, ByteBuffer buf) {
if (logDebug) {
System.out.println("Received DR buffer size " + buf.remaining());
AtomicLong haveOpenTransaction = haveOpenTransactionLocal.get();
buf.order(ByteOrder.LITTLE_ENDIAN);
// Magic header space for Java for implementing zero copy stuff
buf.position(8);
while (buf.hasRemaining()) {
int startPosition = buf.position();
byte version = buf.get();
int type = buf.get();
int checksum = 0;
if (version != 0) System.out.println("Remaining is " + buf.remaining());
switch (DRRecordType.valueOf(type)) {
case INSERT:
{
// Insert
if (haveOpenTransaction.get() == -1) {
System.out.println("Have insert but no open transaction");
System.exit(-1);
}
final long tableHandle = buf.getLong();
final int lengthPrefix = buf.getInt();
buf.position(buf.position() + lengthPrefix);
checksum = buf.getInt();
System.out.println(
"Version "
+ version
+ " type INSERT table handle "
+ tableHandle
+ " length "
+ lengthPrefix
+ " checksum "
+ checksum);
break;
}
case DELETE:
{
// Delete
if (haveOpenTransaction.get() == -1) {
System.out.println("Have insert but no open transaction");
System.exit(-1);
}
final long tableHandle = buf.getLong();
final int lengthPrefix = buf.getInt();
buf.position(buf.position() + lengthPrefix);
checksum = buf.getInt();
System.out.println(
"Version "
+ version
+ " type DELETE table handle "
+ tableHandle
+ " length "
+ lengthPrefix
+ " checksum "
+ checksum);
break;
}
case UPDATE:
// Update
// System.out.println("Version " + version + " type UPDATE " + checksum " + checksum);
break;
case BEGIN_TXN:
{
// Begin txn
final long txnId = buf.getLong();
final long spHandle = buf.getLong();
if (haveOpenTransaction.get() != -1) {
System.out.println(
"Have open transaction txnid "
+ txnId
+ " spHandle "
+ spHandle
+ " but already open transaction");
System.exit(-1);
}
haveOpenTransaction.set(spHandle);
checksum = buf.getInt();
System.out.println(
"Version "
+ version
+ " type BEGIN_TXN "
+ " txnid "
+ txnId
+ " spHandle "
+ spHandle
+ " checksum "
+ checksum);
break;
}
case END_TXN:
{
// End txn
final long spHandle = buf.getLong();
if (haveOpenTransaction.get() == -1) {
System.out.println(
"Have end transaction spHandle "
+ spHandle
+ " but no open transaction and its less then last committed "
+ lastCommittedSpHandle.get().get());
// checksum = buf.getInt();
// break;
System.exit(-1);
}
haveOpenTransaction.set(-1);
lastCommittedSpHandle.get().set(spHandle);
checksum = buf.getInt();
System.out.println(
"Version "
+ version
+ " type END_TXN "
+ " spHandle "
+ spHandle
+ " checksum "
+ checksum);
break;
}
}
int calculatedChecksum =
DBBPool.getBufferCRC32C(buf, startPosition, buf.position() - startPosition - 4);
if (calculatedChecksum != checksum) {
System.out.println("Checksum " + calculatedChecksum + " didn't match " + checksum);
System.exit(-1);
}
}
}
final BBContainer cont = DBBPool.wrapBB(buf);
DBBPool.registerUnsafeMemory(cont.address());
cont.discard();
}
@Override
public void write(byte[] b, int off, int len) throws IOException {
growIfNeeded(len);
buffer.b().put(b, off, len);
}