public void truncateExportToTxnId(long txnId) { try { synchronized (m_committedBuffers) { m_committedBuffers.truncateToTxnId(txnId, m_nullArrayLength); if (m_committedBuffers.isEmpty() && m_endOfStream) { try { m_onDrain.run(); } finally { m_onDrain = null; } } } } catch (IOException e) { VoltDB.crashLocalVoltDB(e.getMessage(), true, e); } }
public void close() { synchronized (m_committedBuffers) { try { m_committedBuffers.close(); } catch (IOException e) { exportLog.error(e); } } }
private void releaseExportBytes(long releaseOffset, ArrayList<StreamBlock> blocksToDelete) throws IOException { // if released offset is in an already-released past, just return success if (!m_committedBuffers.isEmpty() && releaseOffset < m_committedBuffers.peek().uso()) { return; } long lastUso = m_firstUnpolledUso; while (!m_committedBuffers.isEmpty() && releaseOffset >= m_committedBuffers.peek().uso()) { StreamBlock sb = m_committedBuffers.peek(); if (releaseOffset >= sb.uso() + sb.totalUso()) { m_committedBuffers.pop(); blocksToDelete.add(sb); lastUso = sb.uso() + sb.totalUso(); } else if (releaseOffset >= sb.uso()) { sb.releaseUso(releaseOffset); lastUso = releaseOffset; break; } } m_firstUnpolledUso = Math.max(m_firstUnpolledUso, lastUso); }
public void closeAndDelete() throws IOException { m_committedBuffers.closeAndDelete(); }
public void pushExportBuffer( long uso, final long bufferPtr, ByteBuffer buffer, boolean sync, boolean endOfStream) { final java.util.concurrent.atomic.AtomicBoolean deleted = new java.util.concurrent.atomic.AtomicBoolean(false); synchronized (m_committedBuffers) { if (endOfStream) { assert (!m_endOfStream); assert (bufferPtr == 0); assert (buffer == null); assert (!sync); m_endOfStream = endOfStream; if (m_committedBuffers.sizeInBytes() == 0) { exportLog.info("Pushed EOS buffer with 0 bytes remaining"); try { m_onDrain.run(); } finally { m_onDrain = null; } } return; } assert (!m_endOfStream); if (buffer != null) { if (buffer.capacity() > 0) { try { m_committedBuffers.offer( new StreamBlock( new BBContainer(buffer, bufferPtr) { @Override public void discard() { DBBPool.deleteCharArrayMemory(address); deleted.set(true); } }, uso, false)); } catch (IOException e) { exportLog.error(e); if (!deleted.get()) { DBBPool.deleteCharArrayMemory(bufferPtr); } } } else { /* * TupleStreamWrapper::setBytesUsed propagates the USO by sending * over an empty stream block. The block will be deleted * on the native side when this method returns */ exportLog.info( "Syncing first unpolled USO to " + uso + " for table " + m_tableName + " partition " + m_partitionId); m_firstUnpolledUso = uso; } } if (sync) { try { // Don't do a real sync, just write the in memory buffers // to a file. @Quiesce or blocking snapshot will do the sync m_committedBuffers.sync(true); } catch (IOException e) { exportLog.error(e); } } } }
public long sizeInBytes() { return m_committedBuffers.sizeInBytes(); }
/** * Obtain next block of data from source * * @throws MessagingException */ public void exportAction(RawProcessor.ExportInternalMessage m) throws MessagingException { assert (m.m_m.getGeneration() == m_generation); ExportProtoMessage message = m.m_m; ExportProtoMessage result = new ExportProtoMessage(message.getGeneration(), message.m_partitionId, message.m_signature); ExportInternalMessage mbp = new ExportInternalMessage(m.m_sb, result); StreamBlock first_unpolled_block = null; // Assemble a list of blocks to delete so that they can be deleted // outside of the m_committedBuffers critical section ArrayList<StreamBlock> blocksToDelete = new ArrayList<StreamBlock>(); boolean hitEndOfStreamWithNoRunnable = false; try { // Perform all interaction with m_committedBuffers under lock // because pushExportBuffer may be called from an ExecutionSite at any time synchronized (m_committedBuffers) { // Process the ack if any and add blocks to the delete list or move the released USO pointer if (message.isAck() && message.getAckOffset() > 0) { try { releaseExportBytes(message.getAckOffset(), blocksToDelete); } catch (IOException e) { VoltDB.crashLocalVoltDB("Error attempting to release export bytes", true, e); return; } } if (m_endOfStream && m_committedBuffers.sizeInBytes() == 0) { if (m_onDrain != null) { try { m_onDrain.run(); } finally { m_onDrain = null; } } else { hitEndOfStreamWithNoRunnable = true; } return; } // Reset the first unpolled uso so that blocks that have already been polled will // be served up to the next connection if (message.isClose()) { try { resetPollMarker(); } catch (IOException e) { exportLog.error(e); } } // Inside this critical section do the work to find out // what block should be returned by the next poll. // Copying and sending the data will take place outside the critical section try { if (message.isPoll()) { Iterator<StreamBlock> iter = m_committedBuffers.iterator(); while (iter.hasNext()) { StreamBlock block = iter.next(); // find the first block that has unpolled data if (m_firstUnpolledUso < block.uso() + block.totalUso()) { first_unpolled_block = block; m_firstUnpolledUso = block.uso() + block.totalUso(); break; } else { blocksToDelete.add(block); iter.remove(); } } } } catch (RuntimeException e) { if (e.getCause() instanceof IOException) { VoltDB.crashLocalVoltDB("Error attempting to find unpolled export data", true, e); } else { throw e; } } } } finally { // Try hard not to leak memory for (StreamBlock sb : blocksToDelete) { sb.deleteContent(); } // Cheesy hack for now where we serve info about old // data sources from previous generations. In reality accessing // this generation is something of an error if (hitEndOfStreamWithNoRunnable) { ByteBuffer buf = ByteBuffer.allocate(4); buf.putInt(0).flip(); result.pollResponse(m_firstUnpolledUso, buf); mbp.m_sb.event(result); } } if (message.isPoll()) { // If there are no unpolled blocks return the firstUnpolledUSO with no data if (first_unpolled_block == null) { ByteBuffer buf = ByteBuffer.allocate(4); buf.putInt(0).flip(); result.pollResponse(m_firstUnpolledUso, buf); } else { // Otherwise return the block with the USO for the end of the block // since the entire remainder of the block is being sent. result.pollResponse( first_unpolled_block.uso() + first_unpolled_block.totalUso(), first_unpolled_block.unreleasedBuffer()); } mbp.m_sb.event(result); } }
private void resetPollMarker() throws IOException { if (!m_committedBuffers.isEmpty()) { StreamBlock oldestBlock = m_committedBuffers.peek(); m_firstUnpolledUso = oldestBlock.unreleasedUso(); } }