/*
   * this.values = copy.values; this.table = copy.table; this.state = copy.state; this.freeEntries =
   * copy.freeEntries; this.distinct = copy.distinct; this.lowWaterMark = copy.lowWaterMark;
   * this.highWaterMark = copy.highWaterMark; this.minLoadFactor = copy.minLoadFactor;
   * this.maxLoadFactor = copy.maxLoadFactor;
   */
  @Override
  public void write(DataOutput out) throws IOException {
    this.trimToSize();

    WritableUtils.writeVInt(out, freeEntries);
    WritableUtils.writeVInt(out, distinct);
    WritableUtils.writeVInt(out, lowWaterMark);
    WritableUtils.writeVInt(out, highWaterMark);
    out.writeDouble(minLoadFactor);
    out.writeDouble(maxLoadFactor);

    WritableUtils.writeVInt(out, table.length);
    for (int i = 0; i < table.length; i++) {
      WritableUtils.writeVInt(out, table[i]);
    }

    WritableUtils.writeVInt(out, values.length);
    for (int i = 0; i < values.length; i++) {
      out.writeDouble(values[i]);
    }

    WritableUtils.writeVInt(out, state.length);
    for (int i = 0; i < state.length; i++) {
      out.writeByte(state[i]);
    }
  }
Esempio n. 2
0
 @Override
 public void readFields(DataInput input) throws IOException {
   super.readFields(input);
   formatString = WritableUtils.readString(input);
   type = WritableUtils.readEnum(input, FunctionArgumentType.class);
   formatter = type.getFormatter(formatString);
 }
Esempio n. 3
0
 @Override
 public void readFields(DataInput input) throws IOException {
   int encodedByteLengthAndBool = WritableUtils.readVInt(input);
   int byteLength = Math.abs(encodedByteLengthAndBool) - 1;
   this.byteValue = new byte[byteLength];
   input.readFully(byteValue, 0, byteLength);
   int sortOrderAndDeterminism = WritableUtils.readVInt(input);
   if (sortOrderAndDeterminism <= 2) {
     // client is on an older version
     this.determinism = encodedByteLengthAndBool > 0 ? Determinism.ALWAYS : Determinism.PER_ROW;
     this.sortOrder = SortOrder.fromSystemValue(sortOrderAndDeterminism);
     ;
   } else {
     int determinismOrdinal = (sortOrderAndDeterminism >> 2) - 1;
     this.determinism = Determinism.values()[determinismOrdinal];
     int sortOrderValue =
         sortOrderAndDeterminism & ((1 << 2) - 1); // get the least 2 significant bits
     this.sortOrder = SortOrder.fromSystemValue(sortOrderValue);
   }
   int typeOrdinal = WritableUtils.readVInt(input);
   if (typeOrdinal < 0) {
     this.type = null;
   } else {
     this.type = PDataType.values()[typeOrdinal];
   }
   if (this.byteValue.length == 0) {
     this.value = null;
   } else {
     this.value = this.type.toObject(byteValue, 0, byteValue.length, this.type, sortOrder);
   }
 }
Esempio n. 4
0
 @Override
 public void write(DataOutput output) throws IOException {
   super.write(output);
   WritableUtils.writeVInt(output, toType.ordinal());
   WritableUtils.writeVInt(output, ColumnModifier.toSystemValue(toMod));
   WritableUtils.writeVInt(output, byteSize == null ? -1 : byteSize);
 }
  private static ArrayListWritable<PairOfInts> deserializePosting(BytesWritable inputBytes) {

    ArrayListWritable<PairOfInts> posting = new ArrayListWritable<PairOfInts>();
    DataInputStream dataIn = new DataInputStream(new ByteArrayInputStream(inputBytes.getBytes()));
    int prevDocID = 0;

    try {
      while (true) {
        int left = WritableUtils.readVInt(dataIn);
        int right = WritableUtils.readVInt(dataIn);

        if (right != 0) {
          posting.add(new PairOfInts(left + prevDocID, right));
          prevDocID += left;
        }
      }
    } catch (EOFException e) {
    } catch (IOException e) {
    }

    try {
      dataIn.close();
    } catch (IOException e) {
    }

    return posting;
  }
Esempio n. 6
0
 public static TupleProjector deserializeProjectorFromScan(Scan scan) {
   byte[] proj = scan.getAttribute(SCAN_PROJECTOR);
   if (proj == null) {
     return null;
   }
   ByteArrayInputStream stream = new ByteArrayInputStream(proj);
   try {
     DataInputStream input = new DataInputStream(stream);
     KeyValueSchema schema = new KeyValueSchema();
     schema.readFields(input);
     int count = WritableUtils.readVInt(input);
     Expression[] expressions = new Expression[count];
     for (int i = 0; i < count; i++) {
       int ordinal = WritableUtils.readVInt(input);
       expressions[i] = ExpressionType.values()[ordinal].newInstance();
       expressions[i].readFields(input);
     }
     return new TupleProjector(schema, expressions);
   } catch (IOException e) {
     throw new RuntimeException(e);
   } finally {
     try {
       stream.close();
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
   }
 }
  @Override
  public void readFields(DataInput in) throws IOException {

    first_key = WritableUtils.readString(in);
    second_key = WritableUtils.readString(in);
    third_key = WritableUtils.readString(in);
  }
 /**
  * @param cell
  * @param out
  * @param encodingCtx
  * @return unencoded size added
  * @throws IOException
  */
 protected final int afterEncodingKeyValue(
     Cell cell, DataOutputStream out, HFileBlockDefaultEncodingContext encodingCtx)
     throws IOException {
   int size = 0;
   if (encodingCtx.getHFileContext().isIncludesTags()) {
     int tagsLength = cell.getTagsLength();
     ByteBufferUtils.putCompressedInt(out, tagsLength);
     // There are some tags to be written
     if (tagsLength > 0) {
       TagCompressionContext tagCompressionContext = encodingCtx.getTagCompressionContext();
       // When tag compression is enabled, tagCompressionContext will have a not null value. Write
       // the tags using Dictionary compression in such a case
       if (tagCompressionContext != null) {
         tagCompressionContext.compressTags(
             out, cell.getTagsArray(), cell.getTagsOffset(), tagsLength);
       } else {
         out.write(cell.getTagsArray(), cell.getTagsOffset(), tagsLength);
       }
     }
     size += tagsLength + KeyValue.TAGS_LENGTH_SIZE;
   }
   if (encodingCtx.getHFileContext().isIncludesMvcc()) {
     // Copy memstore timestamp from the byte buffer to the output stream.
     long memstoreTS = cell.getSequenceId();
     WritableUtils.writeVLong(out, memstoreTS);
     // TODO use a writeVLong which returns the #bytes written so that 2 time parsing can be
     // avoided.
     size += WritableUtils.getVIntSize(memstoreTS);
   }
   return size;
 }
Esempio n. 9
0
  public void readFields(DataInput in) throws IOException {
    in.readFully(internalBytes, 0, 4);
    if (TimestampWritable.hasDecimalOrSecondVInt(internalBytes[0])) {
      in.readFully(internalBytes, 4, 1);
      int len = (byte) WritableUtils.decodeVIntSize(internalBytes[4]);
      if (len > 1) {
        in.readFully(internalBytes, 5, len - 1);
      }

      long vlong = LazyBinaryUtils.readVLongFromByteArray(internalBytes, 4);
      if (vlong < -1000000000 || vlong > 999999999) {
        throw new IOException(
            "Invalid first vint value (encoded nanoseconds) of a TimestampWritable: "
                + vlong
                + ", expected to be between -1000000000 and 999999999.");
        // Note that -1000000000 is a valid value corresponding to a nanosecond timestamp
        // of 999999999, because if the second VInt is present, we use the value
        // (-reversedNanoseconds - 1) as the second VInt.
      }
      if (vlong < 0) {
        // This indicates there is a second VInt containing the additional bits of the seconds
        // field.
        in.readFully(internalBytes, 4 + len, 1);
        int secondVIntLen = (byte) WritableUtils.decodeVIntSize(internalBytes[4 + len]);
        if (secondVIntLen > 1) {
          in.readFully(internalBytes, 5 + len, secondVIntLen - 1);
        }
      }
    }
    currentBytes = internalBytes;
    this.offset = 0;
  }
Esempio n. 10
0
 public int getEstimatedByteSize() {
   int size = 0;
   size += WritableUtils.getVIntSize(minNullable);
   size += WritableUtils.getVIntSize(fields.size());
   size += fields.size() * 3;
   return size;
 }
Esempio n. 11
0
 /**
  * Write a Map<String, String>
  *
  * @param out DataOutput
  * @param map Map to write
  * @throws IOException I/O errors
  */
 public static void writeStrStrMap(DataOutput out, Map<String, String> map) throws IOException {
   out.writeInt(map.size());
   for (Map.Entry<String, String> entry : map.entrySet()) {
     WritableUtils.writeString(out, entry.getKey());
     WritableUtils.writeString(out, entry.getValue());
   }
 }
Esempio n. 12
0
 public void serialize(DataOutput output) throws IOException {
   WritableUtils.writeVInt(output, minNullable);
   WritableUtils.writeVInt(output, fields.size());
   for (int i = 0; i < fields.size(); i++) {
     fields.get(i).write(output);
   }
 }
  @Override
  public void write(DataOutput out) throws IOException {

    WritableUtils.writeString(out, first_key);
    WritableUtils.writeString(out, second_key);
    WritableUtils.writeString(out, third_key);
  }
Esempio n. 14
0
 @Override
 public void write(DataOutput output) throws IOException {
   WritableUtils.writeVInt(output, (type.ordinal() + 1) * (this.isNullable ? -1 : 1));
   WritableUtils.writeVInt(output, count * (sortOrder == SortOrder.ASC ? 1 : -1));
   if (type.isFixedWidth() && type.getByteSize() == null) {
     WritableUtils.writeVInt(output, byteSize);
   }
 }
Esempio n. 15
0
 @Override
 public void readFields(DataInput input) throws IOException {
   super.readFields(input);
   toType = PDataType.values()[WritableUtils.readVInt(input)];
   toMod = ColumnModifier.fromSystemValue(WritableUtils.readVInt(input));
   int byteSize = WritableUtils.readVInt(input);
   this.byteSize = byteSize == -1 ? null : byteSize;
 }
Esempio n. 16
0
    /* Receive a response.
     * Because only one receiver, so no synchronization on in.
     */
    protected void receiveResponse() {
      if (shouldCloseConnection.get()) {
        return;
      }
      touch();

      try {
        // See HBaseServer.Call.setResponse for where we write out the response.
        // It writes the call.id (int), a flag byte, then optionally the length
        // of the response (int) followed by data.

        // Read the call id.
        int id = in.readInt();

        if (LOG.isDebugEnabled()) LOG.debug(getName() + " got value #" + id);
        Call call = calls.get(id);

        // Read the flag byte
        byte flag = in.readByte();
        boolean isError = ResponseFlag.isError(flag);
        if (ResponseFlag.isLength(flag)) {
          // Currently length if present is unused.
          in.readInt();
        }
        int state = in.readInt(); // Read the state.  Currently unused.
        if (isError) {
          if (call != null) {
            //noinspection ThrowableInstanceNeverThrown
            call.setException(
                new RemoteException(WritableUtils.readString(in), WritableUtils.readString(in)));
          }
        } else {
          Writable value = ReflectionUtils.newInstance(valueClass, conf);
          value.readFields(in); // read value
          // it's possible that this call may have been cleaned up due to a RPC
          // timeout, so check if it still exists before setting the value.
          if (call != null) {
            call.setValue(value);
          }
        }
        calls.remove(id);
      } catch (IOException e) {
        if (e instanceof SocketTimeoutException && remoteId.rpcTimeout > 0) {
          // Clean up open calls but don't treat this as a fatal condition,
          // since we expect certain responses to not make it by the specified
          // {@link ConnectionId#rpcTimeout}.
          closeException = e;
        } else {
          // Since the server did not respond within the default ping interval
          // time, treat this as a fatal condition and close this connection
          markClosed(e);
        }
      } finally {
        if (remoteId.rpcTimeout > 0) {
          cleanupCalls(remoteId.rpcTimeout);
        }
      }
    }
Esempio n. 17
0
 /**
  * Write key-value pair at the head of this stream to the objects provided; get next key-value
  * pair from proxied RR.
  */
 public boolean next(K key, U value) throws IOException {
   if (hasNext()) {
     WritableUtils.cloneInto(key, khead);
     WritableUtils.cloneInto(value, vhead);
     next();
     return true;
   }
   return false;
 }
Esempio n. 18
0
 /**
  * Read in a Map<String, String>
  *
  * @param in DataInput
  * @param map Map to read into
  * @throws IOException I/O errors
  */
 public static void readStrStrMap(DataInput in, Map<String, String> map) throws IOException {
   int size = in.readInt();
   map.clear();
   for (int i = 0; i < size; ++i) {
     String key = WritableUtils.readString(in);
     String value = WritableUtils.readString(in);
     map.put(key, value);
   }
 }
 /** Decode Writable postings and compare by id */
 public int compare(byte[] arg0, int arg1, int arg2, byte[] arg3, int arg4, int arg5) {
   try {
     DataInputStream di1 = new DataInputStream(new ByteArrayInputStream(arg0, arg1, arg2));
     DataInputStream di2 = new DataInputStream(new ByteArrayInputStream(arg3, arg4, arg5));
     return WritableUtils.readVInt(di1) - WritableUtils.readVInt(di2);
   } catch (Exception e) {
     return 0;
   }
 }
Esempio n. 20
0
 public static int getTotalLength(byte[] bytes, int offset) {
   int len = 4;
   if (hasDecimalOrSecondVInt(bytes[offset])) {
     int firstVIntLen = WritableUtils.decodeVIntSize(bytes[offset + 4]);
     len += firstVIntLen;
     if (hasSecondVInt(bytes[offset + 4])) {
       len += WritableUtils.decodeVIntSize(bytes[offset + 4 + firstVIntLen]);
     }
   }
   return len;
 }
    @Override
    public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {

      try {
        int firstL1 = WritableUtils.decodeVIntSize(b1[s1]) + readVInt(b1, s1);
        int firstL2 = WritableUtils.decodeVIntSize(b2[s2]) + readVInt(b2, s2);
        return TEXT_COMPARATOR.compare(b1, s1, firstL1, b2, s2, firstL2);
      } catch (IOException e) {
        throw new IllegalArgumentException(e);
      }
    }
Esempio n. 22
0
 @Override
 public void readFields(DataInput in) throws IOException {
   int minNullable = WritableUtils.readVInt(in);
   int nFields = WritableUtils.readVInt(in);
   List<Field> fields = Lists.newArrayListWithExpectedSize(nFields);
   for (int i = 0; i < nFields; i++) {
     Field field = new Field();
     field.readFields(in);
     fields.add(field);
   }
   init(minNullable, fields);
 }
Esempio n. 23
0
    /** Optimization hook. */
    public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
      long thisLeftValue = readLong(b1, s1);
      long thatLeftValue = readLong(b2, s2);

      if (thisLeftValue == thatLeftValue) {
        int n1 = WritableUtils.decodeVIntSize(b1[s1 + 8]);
        int n2 = WritableUtils.decodeVIntSize(b2[s2 + 8]);
        return compareBytes(b1, s1 + 8 + n1, l1 - n1 - 8, b2, s2 + n2 + 8, l2 - n2 - 8);
      }

      return thisLeftValue < thatLeftValue ? -1 : 1;
    }
 @Override
 public void write(DataOutput out) throws IOException {
   WritableUtils.writeVInt(out, id);
   WritableUtils.writeVLong(out, expirationDate);
   if (secret == null) {
     WritableUtils.writeVInt(out, -1);
   } else {
     byte[] keyBytes = secret.getEncoded();
     WritableUtils.writeVInt(out, keyBytes.length);
     out.write(keyBytes);
   }
 }
Esempio n. 25
0
 @Override
 public void write(DataOutput output) throws IOException {
   WritableUtils.writeVInt(
       output, (byteValue.length + 1) * (this.determinism == Determinism.ALWAYS ? 1 : -1));
   output.write(byteValue);
   // since we need to support clients of a lower version, serialize the determinism enum ordinal
   // in the int used to
   // serialize sort order system value (which is either 1 or 2)
   int sortOrderAndDeterminism =
       ((this.determinism.ordinal() + 1) << 2) + sortOrder.getSystemValue();
   WritableUtils.writeVInt(output, sortOrderAndDeterminism);
   WritableUtils.writeVInt(output, this.type == null ? -1 : this.type.ordinal());
 }
 @Override
 public void readFields(DataInput in) throws IOException {
   id = WritableUtils.readVInt(in);
   expirationDate = WritableUtils.readVLong(in);
   int keyLength = WritableUtils.readVInt(in);
   if (keyLength < 0) {
     secret = null;
   } else {
     byte[] keyBytes = new byte[keyLength];
     in.readFully(keyBytes);
     secret = AuthenticationTokenSecretManager.createSecretKey(keyBytes);
   }
 }
 /**
  * Read a zero-compressed encoded long from a byte array.
  *
  * @param bytes the byte array
  * @param offset the offset in the byte array where the VLong is stored
  * @return the long
  */
 public static long readVLongFromByteArray(final byte[] bytes, int offset) {
   byte firstByte = bytes[offset++];
   int len = WritableUtils.decodeVIntSize(firstByte);
   if (len == 1) {
     return firstByte;
   }
   long i = 0;
   for (int idx = 0; idx < len - 1; idx++) {
     byte b = bytes[offset++];
     i = i << 8;
     i = i | (b & 0xFF);
   }
   return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
 }
Esempio n. 28
0
 /** Similar to {@link WritableUtils#readVLong(DataInput)} but reads from a {@link ByteBuff}. */
 public static long readVLong(ByteBuff in) {
   byte firstByte = in.get();
   int len = WritableUtils.decodeVIntSize(firstByte);
   if (len == 1) {
     return firstByte;
   }
   long i = 0;
   for (int idx = 0; idx < len - 1; idx++) {
     byte b = in.get();
     i = i << 8;
     i = i | (b & 0xFF);
   }
   return (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i);
 }
Esempio n. 29
0
 @Override
 public void write(DataOutput output) throws IOException {
   Bytes.writeByteArray(output, name.getBytes());
   WritableUtils.writeVInt(output, type.ordinal());
   WritableUtils.writeVLong(output, sequenceNumber);
   output.writeLong(timeStamp);
   Bytes.writeByteArray(
       output, pkName == null ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(pkName));
   WritableUtils.writeVInt(output, allColumns.size());
   for (int i = 0; i < allColumns.size(); i++) {
     PColumn column = allColumns.get(i);
     column.write(output);
   }
   stats.write(output);
 }
 @Override
 public void readFields(DataInput input) throws IOException {
   // read/write type ordinal, maxLength presence, scale presence and isNullable bit together to
   // save space
   int typeAndFlag = WritableUtils.readVInt(input);
   isNullable = (typeAndFlag & 0x01) != 0;
   if ((typeAndFlag & 0x02) != 0) {
     scale = WritableUtils.readVInt(input);
   }
   if ((typeAndFlag & 0x04) != 0) {
     maxLength = WritableUtils.readVInt(input);
   }
   type = PDataType.values()[typeAndFlag >>> 3];
   sortOrder = SortOrder.fromSystemValue(WritableUtils.readVInt(input));
 }