Esempio n. 1
0
  public void readFields(DataInput in) throws IOException {
    in.readFully(internalBytes, 0, 4);
    if (TimestampWritable.hasDecimalOrSecondVInt(internalBytes[0])) {
      in.readFully(internalBytes, 4, 1);
      int len = (byte) WritableUtils.decodeVIntSize(internalBytes[4]);
      if (len > 1) {
        in.readFully(internalBytes, 5, len - 1);
      }

      long vlong = LazyBinaryUtils.readVLongFromByteArray(internalBytes, 4);
      if (vlong < -1000000000 || vlong > 999999999) {
        throw new IOException(
            "Invalid first vint value (encoded nanoseconds) of a TimestampWritable: "
                + vlong
                + ", expected to be between -1000000000 and 999999999.");
        // Note that -1000000000 is a valid value corresponding to a nanosecond timestamp
        // of 999999999, because if the second VInt is present, we use the value
        // (-reversedNanoseconds - 1) as the second VInt.
      }
      if (vlong < 0) {
        // This indicates there is a second VInt containing the additional bits of the seconds
        // field.
        in.readFully(internalBytes, 4 + len, 1);
        int secondVIntLen = (byte) WritableUtils.decodeVIntSize(internalBytes[4 + len]);
        if (secondVIntLen > 1) {
          in.readFully(internalBytes, 5 + len, secondVIntLen - 1);
        }
      }
    }
    currentBytes = internalBytes;
    this.offset = 0;
  }
    @Override
    public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {

      try {
        int firstL1 = WritableUtils.decodeVIntSize(b1[s1]) + readVInt(b1, s1);
        int firstL2 = WritableUtils.decodeVIntSize(b2[s2]) + readVInt(b2, s2);
        return TEXT_COMPARATOR.compare(b1, s1, firstL1, b2, s2, firstL2);
      } catch (IOException e) {
        throw new IllegalArgumentException(e);
      }
    }
Esempio n. 3
0
 public static int getTotalLength(byte[] bytes, int offset) {
   int len = 4;
   if (hasDecimalOrSecondVInt(bytes[offset])) {
     int firstVIntLen = WritableUtils.decodeVIntSize(bytes[offset + 4]);
     len += firstVIntLen;
     if (hasSecondVInt(bytes[offset + 4])) {
       len += WritableUtils.decodeVIntSize(bytes[offset + 4 + firstVIntLen]);
     }
   }
   return len;
 }
Esempio n. 4
0
    /** Optimization hook. */
    public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
      long thisLeftValue = readLong(b1, s1);
      long thatLeftValue = readLong(b2, s2);

      if (thisLeftValue == thatLeftValue) {
        int n1 = WritableUtils.decodeVIntSize(b1[s1 + 8]);
        int n2 = WritableUtils.decodeVIntSize(b2[s2 + 8]);
        return compareBytes(b1, s1 + 8 + n1, l1 - n1 - 8, b2, s2 + n2 + 8, l2 - n2 - 8);
      }

      return thisLeftValue < thatLeftValue ? -1 : 1;
    }
Esempio n. 5
0
 public static final int compare(byte[] x, int xs, int xl, byte[] y, int ys, int yl, int[] size) {
   try {
     size[0] = 1 + WritableUtils.decodeVIntSize(x[xs]);
     long v = WritableComparator.readVLong(x, xs) - WritableComparator.readVLong(y, ys);
     return (v == 0) ? 0 : ((v > 0) ? 1 : -1);
   } catch (IOException e) {
     throw new Error(e);
   }
 }
 @Override
 public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
   try {
     int result = 0;
     int pos1 = WritableUtils.decodeVIntSize(b1[s1]);
     int pos2 = WritableUtils.decodeVIntSize(b2[s2]);
     int split1 = WritableComparator.readVInt(b1, s1);
     int split2 = WritableComparator.readVInt(b2, s2);
     result = split2 == split1 ? 0 : (split2 < split1 ? 1 : -1);
     if (result != 0) {
       return result;
     }
     long offset1 = WritableComparator.readVLong(b1, s1 + pos1);
     long offset2 = WritableComparator.readVLong(b2, s2 + pos2);
     result = offset2 == offset1 ? 0 : (offset2 < offset1 ? 1 : -1);
     return result;
   } catch (Exception ex) {
     throw new IllegalArgumentException(ex);
   }
 }
    public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
      try {
        int h1 = WritableUtils.decodeVIntSize(b1[s1 + 1]) + 1;
        int h2 = WritableUtils.decodeVIntSize(b2[s2 + 1]) + 1;

        int cmp = SOURCE_COMP.compare(b1, s1 + h1, l1 - h1, b2, s2 + h2, l2 - h2);
        if (cmp != 0) return cmp;

        cmp = TARGET_COMP.compare(b1, s1 + h1, l1 - h1, b2, s2 + h2, l2 - h2);
        if (cmp != 0) return cmp;

        cmp = PrimitiveUtils.compare(b1[s1], b2[s2]);
        if (cmp != 0) return cmp;

        int lhs1 = Math.abs(WritableComparator.readVInt(b1, s1 + 1));
        int lhs2 = Math.abs(WritableComparator.readVInt(b2, s2 + 1));
        return PrimitiveUtils.compare(lhs1, lhs2);
      } catch (IOException e) {
        throw new IllegalArgumentException(e);
      }
    }
Esempio n. 8
0
 /** Similar to {@link WritableUtils#readVLong(DataInput)} but reads from a {@link ByteBuff}. */
 public static long readVLong(ByteBuff in) {
   byte firstByte = in.get();
   int len = WritableUtils.decodeVIntSize(firstByte);
   if (len == 1) {
     return firstByte;
   }
   long i = 0;
   for (int idx = 0; idx < len - 1; idx++) {
     byte b = in.get();
     i = i << 8;
     i = i | (b & 0xFF);
   }
   return (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i);
 }
 /**
  * Read a zero-compressed encoded long from a byte array.
  *
  * @param bytes the byte array
  * @param offset the offset in the byte array where the VLong is stored
  * @return the long
  */
 public static long readVLongFromByteArray(final byte[] bytes, int offset) {
   byte firstByte = bytes[offset++];
   int len = WritableUtils.decodeVIntSize(firstByte);
   if (len == 1) {
     return firstByte;
   }
   long i = 0;
   for (int idx = 0; idx < len - 1; idx++) {
     byte b = bytes[offset++];
     i = i << 8;
     i = i | (b & 0xFF);
   }
   return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
 }
 /**
  * Reads a zero-compressed encoded int from a byte array and returns it.
  *
  * @param bytes the byte array
  * @param offset offset of the array to read from
  * @param vInt storing the deserialized int and its size in byte
  */
 public static void readVInt(byte[] bytes, int offset, VInt vInt) {
   byte firstByte = bytes[offset];
   vInt.length = (byte) WritableUtils.decodeVIntSize(firstByte);
   if (vInt.length == 1) {
     vInt.value = firstByte;
     return;
   }
   int i = 0;
   for (int idx = 0; idx < vInt.length - 1; idx++) {
     byte b = bytes[offset + 1 + idx];
     i = i << 8;
     i = i | (b & 0xFF);
   }
   vInt.value = (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1) : i);
 }
Esempio n. 11
0
  /**
   * Gets seconds stored as integer at bytes[offset]
   *
   * @param bytes
   * @param offset
   * @return the number of seconds
   */
  public static long getSeconds(byte[] bytes, int offset) {
    int lowest31BitsOfSecondsAndFlag = bytesToInt(bytes, offset);
    if (lowest31BitsOfSecondsAndFlag >= 0
        || // the "has decimal or second VInt" flag is not set
        !hasSecondVInt(bytes[offset + 4])) {
      // The entire seconds field is stored in the first 4 bytes.
      return lowest31BitsOfSecondsAndFlag & LOWEST_31_BITS_OF_SEC_MASK;
    }

    // We compose the seconds field from two parts. The lowest 31 bits come from the first four
    // bytes. The higher-order bits come from the second VInt that follows the nanos field.
    return ((long) (lowest31BitsOfSecondsAndFlag & LOWEST_31_BITS_OF_SEC_MASK))
        | (LazyBinaryUtils.readVLongFromByteArray(
                bytes, offset + 4 + WritableUtils.decodeVIntSize(bytes[offset + 4]))
            << 31);
  }
Esempio n. 12
0
 public static void setTimestamp(Timestamp t, byte[] bytes, int offset) {
   boolean hasDecimalOrSecondVInt = hasDecimalOrSecondVInt(bytes[offset]);
   long seconds = (long) TimestampWritable.getSeconds(bytes, offset);
   int nanos = 0;
   if (hasDecimalOrSecondVInt) {
     nanos = TimestampWritable.getNanos(bytes, offset + 4);
     if (hasSecondVInt(bytes[offset + 4])) {
       seconds +=
           LazyBinaryUtils.readVLongFromByteArray(
               bytes, offset + 4 + WritableUtils.decodeVIntSize(bytes[offset + 4]));
     }
   }
   t.setTime(seconds * 1000);
   if (nanos != 0) {
     t.setNanos(nanos);
   }
 }
Esempio n. 13
0
  /**
   * Writes a Timestamp's serialized value to byte array b at the given offset
   *
   * @param timestamp to convert to bytes
   * @param b destination byte array
   * @param offset destination offset in the byte array
   */
  public static void convertTimestampToBytes(Timestamp t, byte[] b, int offset) {
    long millis = t.getTime();
    int nanos = t.getNanos();

    long seconds = millisToSeconds(millis);
    boolean hasSecondVInt = seconds < 0 || seconds > Integer.MAX_VALUE;
    boolean hasDecimal = setNanosBytes(nanos, b, offset + 4, hasSecondVInt);

    int firstInt = (int) seconds;
    if (hasDecimal || hasSecondVInt) {
      firstInt |= DECIMAL_OR_SECOND_VINT_FLAG;
    } else {
      firstInt &= LOWEST_31_BITS_OF_SEC_MASK;
    }
    intToBytes(firstInt, b, offset);

    if (hasSecondVInt) {
      LazyBinaryUtils.writeVLongToByteArray(
          b, offset + 4 + WritableUtils.decodeVIntSize(b[offset + 4]), seconds >> 31);
    }
  }
Esempio n. 14
0
  /**
   * Given a byte[] that has binary sortable data, initialize the internal structures to hold that
   * data
   *
   * @param bytes the byte array that holds the binary sortable representation
   * @param binSortOffset offset of the binary-sortable representation within the buffer.
   */
  public void setBinarySortable(byte[] bytes, int binSortOffset) {
    // Flip the sign bit (and unused bits of the high-order byte) of the seven-byte long back.
    long seconds = readSevenByteLong(bytes, binSortOffset) ^ SEVEN_BYTE_LONG_SIGN_FLIP;
    int nanos = bytesToInt(bytes, binSortOffset + 7);
    int firstInt = (int) seconds;
    boolean hasSecondVInt = seconds < 0 || seconds > Integer.MAX_VALUE;
    if (nanos != 0 || hasSecondVInt) {
      firstInt |= DECIMAL_OR_SECOND_VINT_FLAG;
    } else {
      firstInt &= LOWEST_31_BITS_OF_SEC_MASK;
    }

    intToBytes(firstInt, internalBytes, 0);
    setNanosBytes(nanos, internalBytes, 4, hasSecondVInt);
    if (hasSecondVInt) {
      LazyBinaryUtils.writeVLongToByteArray(
          internalBytes, 4 + WritableUtils.decodeVIntSize(internalBytes[4]), seconds >> 31);
    }

    currentBytes = internalBytes;
    this.offset = 0;
  }
 public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
   int n1 = WritableUtils.decodeVIntSize(b1[s1]);
   int n2 = WritableUtils.decodeVIntSize(b2[s2]);
   return -1 * WritableComparator.compareBytes(b1, s1 + n1, l1 - n1, b2, s2 + n2, l2 - n2);
 }
 /**
  * Check a particular field and set its size and offset in bytes based on the field type and the
  * bytes arrays.
  *
  * <p>For void, boolean, byte, short, int, long, float and double, there is no offset and the size
  * is fixed. For string, map, list, struct, the first four bytes are used to store the size. So
  * the offset is 4 and the size is computed by concating the first four bytes together. The first
  * four bytes are defined with respect to the offset in the bytes arrays. For timestamp, if the
  * first bit is 0, the record length is 4, otherwise a VInt begins at the 5th byte and its length
  * is added to 4.
  *
  * @param objectInspector object inspector of the field
  * @param bytes bytes arrays store the table row
  * @param offset offset of this field
  * @param recordInfo modify this byteinfo object and return it
  */
 public static void checkObjectByteInfo(
     ObjectInspector objectInspector, byte[] bytes, int offset, RecordInfo recordInfo, VInt vInt) {
   Category category = objectInspector.getCategory();
   switch (category) {
     case PRIMITIVE:
       PrimitiveCategory primitiveCategory =
           ((PrimitiveObjectInspector) objectInspector).getPrimitiveCategory();
       switch (primitiveCategory) {
         case VOID:
           recordInfo.elementOffset = 0;
           recordInfo.elementSize = 0;
           break;
         case BOOLEAN:
         case BYTE:
           recordInfo.elementOffset = 0;
           recordInfo.elementSize = 1;
           break;
         case SHORT:
           recordInfo.elementOffset = 0;
           recordInfo.elementSize = 2;
           break;
         case FLOAT:
           recordInfo.elementOffset = 0;
           recordInfo.elementSize = 4;
           break;
         case DOUBLE:
           recordInfo.elementOffset = 0;
           recordInfo.elementSize = 8;
           break;
         case INT:
           recordInfo.elementOffset = 0;
           recordInfo.elementSize = WritableUtils.decodeVIntSize(bytes[offset]);
           break;
         case LONG:
           recordInfo.elementOffset = 0;
           recordInfo.elementSize = WritableUtils.decodeVIntSize(bytes[offset]);
           break;
         case STRING:
           // using vint instead of 4 bytes
           LazyBinaryUtils.readVInt(bytes, offset, vInt);
           recordInfo.elementOffset = vInt.length;
           recordInfo.elementSize = vInt.value;
           break;
         case CHAR:
         case VARCHAR:
           LazyBinaryUtils.readVInt(bytes, offset, vInt);
           recordInfo.elementOffset = vInt.length;
           recordInfo.elementSize = vInt.value;
           break;
         case BINARY:
           // using vint instead of 4 bytes
           LazyBinaryUtils.readVInt(bytes, offset, vInt);
           recordInfo.elementOffset = vInt.length;
           recordInfo.elementSize = vInt.value;
           break;
         case DATE:
           recordInfo.elementOffset = 0;
           recordInfo.elementSize = WritableUtils.decodeVIntSize(bytes[offset]);
           break;
         case TIMESTAMP:
           recordInfo.elementOffset = 0;
           recordInfo.elementSize = TimestampWritable.getTotalLength(bytes, offset);
           break;
         case DECIMAL:
           // using vint instead of 4 bytes
           LazyBinaryUtils.readVInt(bytes, offset, vInt);
           recordInfo.elementOffset = 0;
           recordInfo.elementSize = vInt.length;
           LazyBinaryUtils.readVInt(bytes, offset + vInt.length, vInt);
           recordInfo.elementSize += vInt.length + vInt.value;
           break;
         default:
           {
             throw new RuntimeException("Unrecognized primitive type: " + primitiveCategory);
           }
       }
       break;
     case LIST:
     case MAP:
     case STRUCT:
     case UNION:
       recordInfo.elementOffset = 4;
       recordInfo.elementSize = LazyBinaryUtils.byteArrayToInt(bytes, offset);
       break;
     default:
       {
         throw new RuntimeException("Unrecognized non-primitive type: " + category);
       }
   }
 }