public void truncateExportToTxnId(long txnId) {
try {
synchronized (m_committedBuffers) {
m_committedBuffers.truncateToTxnId(txnId, m_nullArrayLength);
if (m_committedBuffers.isEmpty() && m_endOfStream) {
try {
m_onDrain.run();
} finally {
m_onDrain = null;
}
}
}
} catch (IOException e) {
VoltDB.crashLocalVoltDB(e.getMessage(), true, e);
}
}
public void close() {
synchronized (m_committedBuffers) {
try {
m_committedBuffers.close();
} catch (IOException e) {
exportLog.error(e);
}
}
}
private void releaseExportBytes(long releaseOffset, ArrayList<StreamBlock> blocksToDelete)
throws IOException {
// if released offset is in an already-released past, just return success
if (!m_committedBuffers.isEmpty() && releaseOffset < m_committedBuffers.peek().uso()) {
return;
}
long lastUso = m_firstUnpolledUso;
while (!m_committedBuffers.isEmpty() && releaseOffset >= m_committedBuffers.peek().uso()) {
StreamBlock sb = m_committedBuffers.peek();
if (releaseOffset >= sb.uso() + sb.totalUso()) {
m_committedBuffers.pop();
blocksToDelete.add(sb);
lastUso = sb.uso() + sb.totalUso();
} else if (releaseOffset >= sb.uso()) {
sb.releaseUso(releaseOffset);
lastUso = releaseOffset;
break;
}
}
m_firstUnpolledUso = Math.max(m_firstUnpolledUso, lastUso);
}
public void closeAndDelete() throws IOException {
m_committedBuffers.closeAndDelete();
}
public void pushExportBuffer(
long uso, final long bufferPtr, ByteBuffer buffer, boolean sync, boolean endOfStream) {
final java.util.concurrent.atomic.AtomicBoolean deleted =
new java.util.concurrent.atomic.AtomicBoolean(false);
synchronized (m_committedBuffers) {
if (endOfStream) {
assert (!m_endOfStream);
assert (bufferPtr == 0);
assert (buffer == null);
assert (!sync);
m_endOfStream = endOfStream;
if (m_committedBuffers.sizeInBytes() == 0) {
exportLog.info("Pushed EOS buffer with 0 bytes remaining");
try {
m_onDrain.run();
} finally {
m_onDrain = null;
}
}
return;
}
assert (!m_endOfStream);
if (buffer != null) {
if (buffer.capacity() > 0) {
try {
m_committedBuffers.offer(
new StreamBlock(
new BBContainer(buffer, bufferPtr) {
@Override
public void discard() {
DBBPool.deleteCharArrayMemory(address);
deleted.set(true);
}
},
uso,
false));
} catch (IOException e) {
exportLog.error(e);
if (!deleted.get()) {
DBBPool.deleteCharArrayMemory(bufferPtr);
}
}
} else {
/*
* TupleStreamWrapper::setBytesUsed propagates the USO by sending
* over an empty stream block. The block will be deleted
* on the native side when this method returns
*/
exportLog.info(
"Syncing first unpolled USO to "
+ uso
+ " for table "
+ m_tableName
+ " partition "
+ m_partitionId);
m_firstUnpolledUso = uso;
}
}
if (sync) {
try {
// Don't do a real sync, just write the in memory buffers
// to a file. @Quiesce or blocking snapshot will do the sync
m_committedBuffers.sync(true);
} catch (IOException e) {
exportLog.error(e);
}
}
}
}
public long sizeInBytes() {
return m_committedBuffers.sizeInBytes();
}
/**
* Obtain next block of data from source
*
* @throws MessagingException
*/
public void exportAction(RawProcessor.ExportInternalMessage m) throws MessagingException {
assert (m.m_m.getGeneration() == m_generation);
ExportProtoMessage message = m.m_m;
ExportProtoMessage result =
new ExportProtoMessage(message.getGeneration(), message.m_partitionId, message.m_signature);
ExportInternalMessage mbp = new ExportInternalMessage(m.m_sb, result);
StreamBlock first_unpolled_block = null;
// Assemble a list of blocks to delete so that they can be deleted
// outside of the m_committedBuffers critical section
ArrayList<StreamBlock> blocksToDelete = new ArrayList<StreamBlock>();
boolean hitEndOfStreamWithNoRunnable = false;
try {
// Perform all interaction with m_committedBuffers under lock
// because pushExportBuffer may be called from an ExecutionSite at any time
synchronized (m_committedBuffers) {
// Process the ack if any and add blocks to the delete list or move the released USO pointer
if (message.isAck() && message.getAckOffset() > 0) {
try {
releaseExportBytes(message.getAckOffset(), blocksToDelete);
} catch (IOException e) {
VoltDB.crashLocalVoltDB("Error attempting to release export bytes", true, e);
return;
}
}
if (m_endOfStream && m_committedBuffers.sizeInBytes() == 0) {
if (m_onDrain != null) {
try {
m_onDrain.run();
} finally {
m_onDrain = null;
}
} else {
hitEndOfStreamWithNoRunnable = true;
}
return;
}
// Reset the first unpolled uso so that blocks that have already been polled will
// be served up to the next connection
if (message.isClose()) {
try {
resetPollMarker();
} catch (IOException e) {
exportLog.error(e);
}
}
// Inside this critical section do the work to find out
// what block should be returned by the next poll.
// Copying and sending the data will take place outside the critical section
try {
if (message.isPoll()) {
Iterator<StreamBlock> iter = m_committedBuffers.iterator();
while (iter.hasNext()) {
StreamBlock block = iter.next();
// find the first block that has unpolled data
if (m_firstUnpolledUso < block.uso() + block.totalUso()) {
first_unpolled_block = block;
m_firstUnpolledUso = block.uso() + block.totalUso();
break;
} else {
blocksToDelete.add(block);
iter.remove();
}
}
}
} catch (RuntimeException e) {
if (e.getCause() instanceof IOException) {
VoltDB.crashLocalVoltDB("Error attempting to find unpolled export data", true, e);
} else {
throw e;
}
}
}
} finally {
// Try hard not to leak memory
for (StreamBlock sb : blocksToDelete) {
sb.deleteContent();
}
// Cheesy hack for now where we serve info about old
// data sources from previous generations. In reality accessing
// this generation is something of an error
if (hitEndOfStreamWithNoRunnable) {
ByteBuffer buf = ByteBuffer.allocate(4);
buf.putInt(0).flip();
result.pollResponse(m_firstUnpolledUso, buf);
mbp.m_sb.event(result);
}
}
if (message.isPoll()) {
// If there are no unpolled blocks return the firstUnpolledUSO with no data
if (first_unpolled_block == null) {
ByteBuffer buf = ByteBuffer.allocate(4);
buf.putInt(0).flip();
result.pollResponse(m_firstUnpolledUso, buf);
} else {
// Otherwise return the block with the USO for the end of the block
// since the entire remainder of the block is being sent.
result.pollResponse(
first_unpolled_block.uso() + first_unpolled_block.totalUso(),
first_unpolled_block.unreleasedBuffer());
}
mbp.m_sb.event(result);
}
}
private void resetPollMarker() throws IOException {
if (!m_committedBuffers.isEmpty()) {
StreamBlock oldestBlock = m_committedBuffers.peek();
m_firstUnpolledUso = oldestBlock.unreleasedUso();
}
}
