Exemplo n.º 1
0
    private void convertChunks() throws IOException, InterruptedException {
      int lastNumCharacters = 1024 * 64;
      while (!Thread.interrupted() && m_saveFile.hasMoreChunks()) {
        if (m_availableBytes.get() > m_maxAvailableBytes) {
          Thread.sleep(5);
          continue;
        }

        BBContainer c = m_saveFile.getNextChunk();
        if (c == null) {
          return;
        }

        try {
          final VoltTable vt = PrivateVoltTableFactory.createVoltTableFromBuffer(c.b(), true);
          Pair<Integer, byte[]> p = VoltTableUtil.toCSV(vt, m_delimiter, null, lastNumCharacters);
          lastNumCharacters = p.getFirst();
          byte csvBytes[] = p.getSecond();
          // should not insert empty byte[] if not last ConverterThread
          if (csvBytes.length > 0) {
            m_availableBytes.addAndGet(csvBytes.length);
            m_available.offer(csvBytes);
          }
        } finally {
          c.discard();
        }
      }
    }
Exemplo n.º 2
0
 BBContainer acquire() {
   final BBContainer cont = m_buffers.poll();
   if (cont == null) {
     final BBContainer originContainer = DBBPool.allocateDirect(1024 * 32);
     return new BBContainer(originContainer.b()) {
       @Override
       public void discard() {
         checkDoubleFree();
         // If we had to allocate over the desired limit, start discarding
         if (m_buffers.size() > m_numBuffers) {
           originContainer.discard();
           return;
         }
         m_buffers.push(originContainer);
       }
     };
   }
   return new BBContainer(cont.b()) {
     @Override
     public void discard() {
       checkDoubleFree();
       m_buffers.push(cont);
     }
   };
 }
Exemplo n.º 3
0
  /** Resizes the internal byte buffer with a simple doubling policy, if needed. */
  private final void growIfNeeded(int minimumDesired) {
    if (buffer.b().remaining() < minimumDesired) {
      // Compute the size of the new buffer
      int newCapacity = buffer.b().capacity();
      int newRemaining = newCapacity - buffer.b().position();
      while (newRemaining < minimumDesired) {
        newRemaining += newCapacity;
        newCapacity *= 2;
      }

      // Allocate and copy
      BBContainer next;
      if (isDirect) {
        next = DBBPool.allocateDirect(newCapacity);
      } else {
        next = DBBPool.wrapBB(ByteBuffer.allocate(newCapacity));
      }
      buffer.b().flip();
      next.b().put(buffer.b());
      assert next.b().remaining() == newRemaining;
      buffer.discard();
      buffer = next;
      if (callback != null) callback.onBufferGrow(this);
      assert (buffer.b().order() == ByteOrder.BIG_ENDIAN);
    }
  }
Exemplo n.º 4
0
 /**
  * Return a readOnly slice of this buffer. Flips the internal buffer. May not be, usefully,
  * invoked multiple times on the same internal state.
  *
  * <p>Only use this if using a non-direct ByteBuffer!
  */
 public ByteBuffer getBuffer() {
   assert (isDirect == false);
   assert (buffer.b().hasArray());
   assert (!buffer.b().isDirect());
   buffer.b().flip();
   return buffer.b().asReadOnlyBuffer();
 }
Exemplo n.º 5
0
 /**
  * Get a ascii-string-safe version of the binary value using a hex encoding.
  *
  * @return A hex-encoded string value representing the serialized objects.
  */
 public String getHexEncodedBytes() {
   buffer.b().flip();
   byte bytes[] = new byte[buffer.b().remaining()];
   buffer.b().get(bytes);
   String hex = Encoder.hexEncode(bytes);
   buffer.discard();
   return hex;
 }
Exemplo n.º 6
0
 /**
  * This method is slow and horrible. It entails an extra copy. Don't use it! Ever! Not even for
  * test! Just say no to test only code. It will also leak the BBContainer if this FS is being used
  * with a pool.
  */
 public byte[] getBytes() {
   byte[] retval = new byte[buffer.b().position()];
   int position = buffer.b().position();
   buffer.b().rewind();
   buffer.b().get(retval);
   assert position == buffer.b().position();
   return retval;
 }
Exemplo n.º 7
0
 @Override
 public ListenableFuture<?> write(Callable<BBContainer> tupleData, int tableId) {
   try {
     BBContainer container = tupleData.call();
     if (container != null) {
       container.discard();
     }
   } catch (Exception e) {
   }
   return null;
 }
Exemplo n.º 8
0
 public void writeArray(BigDecimal[] values) throws IOException {
   if (values.length > Short.MAX_VALUE) {
     throw new IOException("Array exceeds maximum length of " + Short.MAX_VALUE + " bytes");
   }
   writeShort(values.length);
   growIfNeeded(16); // sizeof bigdecimal
   for (int i = 0; i < values.length; ++i) {
     if (values[i] == null) {
       VoltDecimalHelper.serializeNull(buffer.b());
     } else {
       VoltDecimalHelper.serializeBigDecimal(values[i], buffer.b());
     }
   }
 }
Exemplo n.º 9
0
 @Override
 public void discard() {
   if (m_hasMoreChunks == false) {
     m_origin.discard();
   } else {
     m_buffers.add(this);
   }
 }
Exemplo n.º 10
0
 /** constructor that sets callback object. */
 public FastSerializer(
     boolean bigEndian, boolean isDirect, BufferGrowCallback callback, int initialAllocation) {
   assert (initialAllocation > 0);
   this.isDirect = isDirect;
   if (isDirect) {
     buffer = DBBPool.allocateDirect(initialAllocation);
   } else {
     buffer = DBBPool.wrapBB(ByteBuffer.allocate(initialAllocation));
   }
   this.callback = callback;
   buffer.b().order(bigEndian ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN);
 }
Exemplo n.º 11
0
  public void close() throws IOException {
    if (m_chunkReaderThread != null) {
      m_chunkReaderThread.interrupt();
      try {
        m_chunkReaderThread.join();
      } catch (InterruptedException e) {
        throw new IOException(e);
      }
    }
    synchronized (this) {
      while (!m_availableChunks.isEmpty()) {
        m_availableChunks.poll().discard();
      }
      notifyAll();
    }

    /*
     * Free buffers used to pull snapshot data in process
     */
    BBContainer cont;
    while ((cont = m_buffers.poll()) != null) {
      cont.discard();
    }
  }
Exemplo n.º 12
0
 /**
  * Set current position of underlying buffer. Useful only in concert with getPosition()
  *
  * @param pos The position to set to.
  */
 public void setPosition(int pos) {
   buffer.b().position(pos);
 }
Exemplo n.º 13
0
 public void discard() {
   buffer.discard();
 }
Exemplo n.º 14
0
 @Override
 public void writeShort(int v) throws IOException {
   growIfNeeded(Short.SIZE / 8);
   buffer.b().putShort((short) v);
 }
Exemplo n.º 15
0
 /** return Current position within the underlying buffer, for self-comparison only. */
 public int getPosition() {
   return buffer.b().position();
 }
Exemplo n.º 16
0
 @Override
 public void writeInt(int v) throws IOException {
   growIfNeeded(Integer.SIZE / 8);
   buffer.b().putInt(v);
 }
Exemplo n.º 17
0
 @Override
 public void writeLong(long v) throws IOException {
   growIfNeeded(Long.SIZE / 8);
   buffer.b().putLong(v);
 }
Exemplo n.º 18
0
 @Override
 public void writeDouble(double v) throws IOException {
   growIfNeeded(Double.SIZE / 8);
   buffer.b().putDouble(v);
 }
Exemplo n.º 19
0
 /** Write an SPI using it's ByteBuffer serialization code. */
 public void writeInvocation(StoredProcedureInvocation invocation) throws IOException {
   int len = invocation.getSerializedSize();
   growIfNeeded(len);
   invocation.flattenToBuffer(buffer.b());
 }
Exemplo n.º 20
0
 @Override
 public void writeByte(int v) throws IOException {
   growIfNeeded(Byte.SIZE / 8);
   buffer.b().put((byte) v);
 }
Exemplo n.º 21
0
 @Override
 public void writeChar(int v) throws IOException {
   growIfNeeded(Character.SIZE / 8);
   buffer.b().putChar((char) v);
 }
Exemplo n.º 22
0
 /**
  * When a fast serializer is shared between Java and native this is called to retrieve a reference
  * to the to buffer without flipping it. OnBufferGrowCallback needs this to update the pointer to
  * the shared buffer when the parameter buffer grows.
  */
 public BBContainer getContainerNoFlip() {
   assert (isDirect == true);
   assert (buffer.b().isDirect());
   return buffer;
 }
Exemplo n.º 23
0
 /** Write a table using it's ByteBuffer serialization code. */
 public void writeTable(VoltTable table) throws IOException {
   int len = table.getSerializedSize();
   growIfNeeded(len);
   table.flattenToBuffer(buffer.b());
 }
Exemplo n.º 24
0
 @Override
 public void write(byte[] b) throws IOException {
   growIfNeeded(b.length);
   buffer.b().put(b);
 }
Exemplo n.º 25
0
 /** @return a reference to the underlying ByteBuffer. */
 public BBContainer getBBContainer() {
   buffer.b().flip();
   return buffer;
 }
Exemplo n.º 26
0
 @Override
 public void writeFloat(float v) throws IOException {
   growIfNeeded(Float.SIZE / 8);
   buffer.b().putFloat(v);
 }
Exemplo n.º 27
0
 public void write(ByteBuffer b) throws IOException {
   growIfNeeded(b.limit() - b.position());
   buffer.b().put(b);
 }
Exemplo n.º 28
0
 /** Write a ParameterSet using it's ByteBuffer serialization code. */
 public void writeParameterSet(ParameterSet params) throws IOException {
   int len = params.getSerializedSize();
   growIfNeeded(len);
   params.flattenToBuffer(buffer.b());
 }
Exemplo n.º 29
0
  public static synchronized void pushDRBuffer(int partitionId, ByteBuffer buf) {
    if (logDebug) {
      System.out.println("Received DR buffer size " + buf.remaining());
      AtomicLong haveOpenTransaction = haveOpenTransactionLocal.get();
      buf.order(ByteOrder.LITTLE_ENDIAN);
      // Magic header space for Java for implementing zero copy stuff
      buf.position(8);
      while (buf.hasRemaining()) {
        int startPosition = buf.position();
        byte version = buf.get();
        int type = buf.get();

        int checksum = 0;
        if (version != 0) System.out.println("Remaining is " + buf.remaining());

        switch (DRRecordType.valueOf(type)) {
          case INSERT:
            {
              // Insert
              if (haveOpenTransaction.get() == -1) {
                System.out.println("Have insert but no open transaction");
                System.exit(-1);
              }
              final long tableHandle = buf.getLong();
              final int lengthPrefix = buf.getInt();
              buf.position(buf.position() + lengthPrefix);
              checksum = buf.getInt();
              System.out.println(
                  "Version "
                      + version
                      + " type INSERT table handle "
                      + tableHandle
                      + " length "
                      + lengthPrefix
                      + " checksum "
                      + checksum);
              break;
            }
          case DELETE:
            {
              // Delete
              if (haveOpenTransaction.get() == -1) {
                System.out.println("Have insert but no open transaction");
                System.exit(-1);
              }
              final long tableHandle = buf.getLong();
              final int lengthPrefix = buf.getInt();
              buf.position(buf.position() + lengthPrefix);
              checksum = buf.getInt();
              System.out.println(
                  "Version "
                      + version
                      + " type DELETE table handle "
                      + tableHandle
                      + " length "
                      + lengthPrefix
                      + " checksum "
                      + checksum);
              break;
            }
          case UPDATE:
            // Update
            // System.out.println("Version " + version + " type UPDATE " + checksum " + checksum);
            break;
          case BEGIN_TXN:
            {
              // Begin txn
              final long txnId = buf.getLong();
              final long spHandle = buf.getLong();
              if (haveOpenTransaction.get() != -1) {
                System.out.println(
                    "Have open transaction txnid "
                        + txnId
                        + " spHandle "
                        + spHandle
                        + " but already open transaction");
                System.exit(-1);
              }
              haveOpenTransaction.set(spHandle);
              checksum = buf.getInt();
              System.out.println(
                  "Version "
                      + version
                      + " type BEGIN_TXN "
                      + " txnid "
                      + txnId
                      + " spHandle "
                      + spHandle
                      + " checksum "
                      + checksum);
              break;
            }
          case END_TXN:
            {
              // End txn
              final long spHandle = buf.getLong();
              if (haveOpenTransaction.get() == -1) {
                System.out.println(
                    "Have end transaction spHandle "
                        + spHandle
                        + " but no open transaction and its less then last committed "
                        + lastCommittedSpHandle.get().get());
                //                    checksum = buf.getInt();
                //                    break;
                System.exit(-1);
              }
              haveOpenTransaction.set(-1);
              lastCommittedSpHandle.get().set(spHandle);
              checksum = buf.getInt();
              System.out.println(
                  "Version "
                      + version
                      + " type END_TXN "
                      + " spHandle "
                      + spHandle
                      + " checksum "
                      + checksum);
              break;
            }
        }
        int calculatedChecksum =
            DBBPool.getBufferCRC32C(buf, startPosition, buf.position() - startPosition - 4);
        if (calculatedChecksum != checksum) {
          System.out.println("Checksum " + calculatedChecksum + " didn't match " + checksum);
          System.exit(-1);
        }
      }
    }
    final BBContainer cont = DBBPool.wrapBB(buf);
    DBBPool.registerUnsafeMemory(cont.address());
    cont.discard();
  }
Exemplo n.º 30
0
 @Override
 public void write(byte[] b, int off, int len) throws IOException {
   growIfNeeded(len);
   buffer.b().put(b, off, len);
 }