Ejemplo n.º 1
0
 /**
  * Return a readOnly slice of this buffer. Flips the internal buffer. May not be, usefully,
  * invoked multiple times on the same internal state.
  *
  * <p>Only use this if using a non-direct ByteBuffer!
  */
 public ByteBuffer getBuffer() {
   assert (isDirect == false);
   assert (buffer.b().hasArray());
   assert (!buffer.b().isDirect());
   buffer.b().flip();
   return buffer.b().asReadOnlyBuffer();
 }
Ejemplo n.º 2
0
 BBContainer acquire() {
   final BBContainer cont = m_buffers.poll();
   if (cont == null) {
     final BBContainer originContainer = DBBPool.allocateDirect(1024 * 32);
     return new BBContainer(originContainer.b()) {
       @Override
       public void discard() {
         checkDoubleFree();
         // If we had to allocate over the desired limit, start discarding
         if (m_buffers.size() > m_numBuffers) {
           originContainer.discard();
           return;
         }
         m_buffers.push(originContainer);
       }
     };
   }
   return new BBContainer(cont.b()) {
     @Override
     public void discard() {
       checkDoubleFree();
       m_buffers.push(cont);
     }
   };
 }
Ejemplo n.º 3
0
 /**
  * Get a ascii-string-safe version of the binary value using a hex encoding.
  *
  * @return A hex-encoded string value representing the serialized objects.
  */
 public String getHexEncodedBytes() {
   buffer.b().flip();
   byte bytes[] = new byte[buffer.b().remaining()];
   buffer.b().get(bytes);
   String hex = Encoder.hexEncode(bytes);
   buffer.discard();
   return hex;
 }
Ejemplo n.º 4
0
 /**
  * This method is slow and horrible. It entails an extra copy. Don't use it! Ever! Not even for
  * test! Just say no to test only code. It will also leak the BBContainer if this FS is being used
  * with a pool.
  */
 public byte[] getBytes() {
   byte[] retval = new byte[buffer.b().position()];
   int position = buffer.b().position();
   buffer.b().rewind();
   buffer.b().get(retval);
   assert position == buffer.b().position();
   return retval;
 }
Ejemplo n.º 5
0
 public void writeArray(BigDecimal[] values) throws IOException {
   if (values.length > Short.MAX_VALUE) {
     throw new IOException("Array exceeds maximum length of " + Short.MAX_VALUE + " bytes");
   }
   writeShort(values.length);
   growIfNeeded(16); // sizeof bigdecimal
   for (int i = 0; i < values.length; ++i) {
     if (values[i] == null) {
       VoltDecimalHelper.serializeNull(buffer.b());
     } else {
       VoltDecimalHelper.serializeBigDecimal(values[i], buffer.b());
     }
   }
 }
Ejemplo n.º 6
0
  /** Resizes the internal byte buffer with a simple doubling policy, if needed. */
  private final void growIfNeeded(int minimumDesired) {
    if (buffer.b().remaining() < minimumDesired) {
      // Compute the size of the new buffer
      int newCapacity = buffer.b().capacity();
      int newRemaining = newCapacity - buffer.b().position();
      while (newRemaining < minimumDesired) {
        newRemaining += newCapacity;
        newCapacity *= 2;
      }

      // Allocate and copy
      BBContainer next;
      if (isDirect) {
        next = DBBPool.allocateDirect(newCapacity);
      } else {
        next = DBBPool.wrapBB(ByteBuffer.allocate(newCapacity));
      }
      buffer.b().flip();
      next.b().put(buffer.b());
      assert next.b().remaining() == newRemaining;
      buffer.discard();
      buffer = next;
      if (callback != null) callback.onBufferGrow(this);
      assert (buffer.b().order() == ByteOrder.BIG_ENDIAN);
    }
  }
Ejemplo n.º 7
0
    private void convertChunks() throws IOException, InterruptedException {
      int lastNumCharacters = 1024 * 64;
      while (!Thread.interrupted() && m_saveFile.hasMoreChunks()) {
        if (m_availableBytes.get() > m_maxAvailableBytes) {
          Thread.sleep(5);
          continue;
        }

        BBContainer c = m_saveFile.getNextChunk();
        if (c == null) {
          return;
        }

        try {
          final VoltTable vt = PrivateVoltTableFactory.createVoltTableFromBuffer(c.b(), true);
          Pair<Integer, byte[]> p = VoltTableUtil.toCSV(vt, m_delimiter, null, lastNumCharacters);
          lastNumCharacters = p.getFirst();
          byte csvBytes[] = p.getSecond();
          // should not insert empty byte[] if not last ConverterThread
          if (csvBytes.length > 0) {
            m_availableBytes.addAndGet(csvBytes.length);
            m_available.offer(csvBytes);
          }
        } finally {
          c.discard();
        }
      }
    }
Ejemplo n.º 8
0
 /** constructor that sets callback object. */
 public FastSerializer(
     boolean bigEndian, boolean isDirect, BufferGrowCallback callback, int initialAllocation) {
   assert (initialAllocation > 0);
   this.isDirect = isDirect;
   if (isDirect) {
     buffer = DBBPool.allocateDirect(initialAllocation);
   } else {
     buffer = DBBPool.wrapBB(ByteBuffer.allocate(initialAllocation));
   }
   this.callback = callback;
   buffer.b().order(bigEndian ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN);
 }
Ejemplo n.º 9
0
 @Override
 public void writeByte(int v) throws IOException {
   growIfNeeded(Byte.SIZE / 8);
   buffer.b().put((byte) v);
 }
Ejemplo n.º 10
0
 @Override
 public void write(byte[] b, int off, int len) throws IOException {
   growIfNeeded(len);
   buffer.b().put(b, off, len);
 }
Ejemplo n.º 11
0
 /** return Current position within the underlying buffer, for self-comparison only. */
 public int getPosition() {
   return buffer.b().position();
 }
Ejemplo n.º 12
0
 @Override
 public void writeLong(long v) throws IOException {
   growIfNeeded(Long.SIZE / 8);
   buffer.b().putLong(v);
 }
Ejemplo n.º 13
0
 @Override
 public void writeFloat(float v) throws IOException {
   growIfNeeded(Float.SIZE / 8);
   buffer.b().putFloat(v);
 }
Ejemplo n.º 14
0
 /** Write a table using it's ByteBuffer serialization code. */
 public void writeTable(VoltTable table) throws IOException {
   int len = table.getSerializedSize();
   growIfNeeded(len);
   table.flattenToBuffer(buffer.b());
 }
Ejemplo n.º 15
0
 BBContainer unreleasedContainer() {
   m_refCount.incrementAndGet();
   return getRefCountingContainer(m_buffer.b().slice().asReadOnlyBuffer());
 }
Ejemplo n.º 16
0
 /**
  * When a fast serializer is shared between Java and native this is called to retrieve a reference
  * to the to buffer without flipping it. OnBufferGrowCallback needs this to update the pointer to
  * the shared buffer when the parameter buffer grows.
  */
 public BBContainer getContainerNoFlip() {
   assert (isDirect == true);
   assert (buffer.b().isDirect());
   return buffer;
 }
Ejemplo n.º 17
0
 /*
  * Does not increment the refcount, uses the implicit 1 count
  * and should only be called once to get a container for pushing the data to disk
  */
 BBContainer asBBContainer() {
   m_buffer.b().putLong(0, uso());
   m_buffer.b().position(0);
   return getRefCountingContainer(m_buffer.b().asReadOnlyBuffer());
 }
Ejemplo n.º 18
0
 public int size() {
   return buffer.b().position();
 }
Ejemplo n.º 19
0
 /** @return a reference to the underlying ByteBuffer. */
 public BBContainer getBBContainer() {
   buffer.b().flip();
   return buffer;
 }
Ejemplo n.º 20
0
 @Override
 public void writeChar(int v) throws IOException {
   growIfNeeded(Character.SIZE / 8);
   buffer.b().putChar((char) v);
 }
Ejemplo n.º 21
0
 @Override
 public void writeDouble(double v) throws IOException {
   growIfNeeded(Double.SIZE / 8);
   buffer.b().putDouble(v);
 }
Ejemplo n.º 22
0
 /** Write an SPI using it's ByteBuffer serialization code. */
 public void writeInvocation(StoredProcedureInvocation invocation) throws IOException {
   int len = invocation.getSerializedSize();
   growIfNeeded(len);
   invocation.flattenToBuffer(buffer.b());
 }
Ejemplo n.º 23
0
 @Override
 public void writeInt(int v) throws IOException {
   growIfNeeded(Integer.SIZE / 8);
   buffer.b().putInt(v);
 }
Ejemplo n.º 24
0
 /** Write a ParameterSet using it's ByteBuffer serialization code. */
 public void writeParameterSet(ParameterSet params) throws IOException {
   int len = params.getSerializedSize();
   growIfNeeded(len);
   params.flattenToBuffer(buffer.b());
 }
Ejemplo n.º 25
0
 @Override
 public void writeShort(int v) throws IOException {
   growIfNeeded(Short.SIZE / 8);
   buffer.b().putShort((short) v);
 }
Ejemplo n.º 26
0
  /**
   * Process a message pulled off from the network thread, and discard the container once it's
   * processed.
   *
   * @param msg A pair of <sourceHSId, blockContainer>
   * @return The restore work, or null if there's no data block to return to the site.
   */
  private RestoreWork processMessage(
      Pair<Long, Pair<Long, BBContainer>> msg, CachedByteBufferAllocator resultBufferAllocator) {
    if (msg == null) {
      return null;
    }

    RestoreWork restoreWork = null;
    long hsId = msg.getFirst();
    long targetId = msg.getSecond().getFirst();
    BBContainer container = msg.getSecond().getSecond();
    try {
      ByteBuffer block = container.b();
      byte typeByte = block.get(StreamSnapshotDataTarget.typeOffset);
      final int blockIndex = block.getInt(StreamSnapshotDataTarget.blockIndexOffset);
      StreamSnapshotMessageType type = StreamSnapshotMessageType.values()[typeByte];
      if (type == StreamSnapshotMessageType.FAILURE) {
        VoltDB.crashLocalVoltDB("Rejoin source sent failure message.", false, null);

        // for test code only
        if (m_expectedEOFs.decrementAndGet() == 0) {
          m_EOF = true;
        }
      } else if (type == StreamSnapshotMessageType.END) {
        if (rejoinLog.isTraceEnabled()) {
          rejoinLog.trace("Got END message " + blockIndex);
        }

        // End of stream, no need to ack this buffer
        if (m_expectedEOFs.decrementAndGet() == 0) {
          m_EOF = true;
        }
      } else if (type == StreamSnapshotMessageType.SCHEMA) {
        rejoinLog.trace("Got SCHEMA message");

        block.position(StreamSnapshotDataTarget.contentOffset);
        byte[] schemaBytes = new byte[block.remaining()];
        block.get(schemaBytes);
        m_schemas.put(block.getInt(StreamSnapshotDataTarget.tableIdOffset), schemaBytes);
      } else if (type == StreamSnapshotMessageType.HASHINATOR) {
        block.position(StreamSnapshotDataTarget.contentOffset);
        long version = block.getLong();
        byte[] hashinatorConfig = new byte[block.remaining()];
        block.get(hashinatorConfig);

        restoreWork = new HashinatorRestoreWork(version, hashinatorConfig);
      } else {
        // It's normal snapshot data afterwards

        final int tableId = block.getInt(StreamSnapshotDataTarget.tableIdOffset);

        if (!m_schemas.containsKey(tableId)) {
          VoltDB.crashLocalVoltDB("No schema for table with ID " + tableId, false, null);
        }

        // Get the byte buffer ready to be consumed
        block.position(StreamSnapshotDataTarget.contentOffset);
        ByteBuffer nextChunk = getNextChunk(m_schemas.get(tableId), block, resultBufferAllocator);
        m_bytesReceived += nextChunk.remaining();

        restoreWork = new TableRestoreWork(tableId, nextChunk);
      }

      // Queue ack to this block
      m_ack.ack(hsId, m_EOF, targetId, blockIndex);

      return restoreWork;
    } finally {
      container.discard();
    }
  }
Ejemplo n.º 27
0
 /**
  * Set current position of underlying buffer. Useful only in concert with getPosition()
  *
  * @param pos The position to set to.
  */
 public void setPosition(int pos) {
   buffer.b().position(pos);
 }
Ejemplo n.º 28
0
 @Override
 public void write(byte[] b) throws IOException {
   growIfNeeded(b.length);
   buffer.b().put(b);
 }
Ejemplo n.º 29
0
 public void write(ByteBuffer b) throws IOException {
   growIfNeeded(b.limit() - b.position());
   buffer.b().put(b);
 }
Ejemplo n.º 30
0
 /** Clears the contents of the underlying buffer, making iteady for more writes. */
 public void clear() {
   buffer.b().clear();
 }