Exemplo n.º 1
0
 @Override
 public void readFields(DataInput input) throws IOException {
   int encodedByteLengthAndBool = WritableUtils.readVInt(input);
   int byteLength = Math.abs(encodedByteLengthAndBool) - 1;
   this.byteValue = new byte[byteLength];
   input.readFully(byteValue, 0, byteLength);
   int sortOrderAndDeterminism = WritableUtils.readVInt(input);
   if (sortOrderAndDeterminism <= 2) {
     // client is on an older version
     this.determinism = encodedByteLengthAndBool > 0 ? Determinism.ALWAYS : Determinism.PER_ROW;
     this.sortOrder = SortOrder.fromSystemValue(sortOrderAndDeterminism);
     ;
   } else {
     int determinismOrdinal = (sortOrderAndDeterminism >> 2) - 1;
     this.determinism = Determinism.values()[determinismOrdinal];
     int sortOrderValue =
         sortOrderAndDeterminism & ((1 << 2) - 1); // get the least 2 significant bits
     this.sortOrder = SortOrder.fromSystemValue(sortOrderValue);
   }
   int typeOrdinal = WritableUtils.readVInt(input);
   if (typeOrdinal < 0) {
     this.type = null;
   } else {
     this.type = PDataType.values()[typeOrdinal];
   }
   if (this.byteValue.length == 0) {
     this.value = null;
   } else {
     this.value = this.type.toObject(byteValue, 0, byteValue.length, this.type, sortOrder);
   }
 }
Exemplo n.º 2
0
 /**
  * Assigns the specified byte values to elements of the specified range of the specified array of
  * bytes. The range to be filled extends from index fromIndex, inclusive, to index toIndex,
  * exclusive. (If fromIndex==toIndex, the range to be filled is empty.)
  *
  * @param str the array to be filled
  * @param strFromIdx the index of the first element (inclusive) to be filled with the fill values
  * @param strToIdx the index of the last element (exclusive) to be filled with the fill values
  * @param fillArray the values to be stored in all elements of the array
  * @param fillFromIdx the index of the first element (inclusive) to be used as fill values
  * @param filToIdx the index of the last element (exclusive) to be used as fill value
  * @param invertFill if true inverts the bits in fill before filling the array
  */
 public static void fill(
     byte[] str,
     int strFromIdx,
     int strToIdx,
     byte[] fillArray,
     int fillFromIdx,
     int fillToIdx,
     boolean invertFill) {
   rangeCheck(str.length, strFromIdx, strToIdx);
   rangeCheck(fillArray.length, fillFromIdx, fillToIdx);
   int strIdx = strFromIdx;
   byte[] fill = fillArray;
   int fillLen = fillToIdx - fillFromIdx;
   if (invertFill) fill = SortOrder.invert(fillArray, fillFromIdx, fillLen);
   while (strIdx < strToIdx) {
     int fillIdx = fillFromIdx;
     while (fillIdx < fillToIdx && strIdx < strToIdx) {
       if (strIdx + fillLen < fillToIdx) {
         System.arraycopy(fill, fillFromIdx, str, strIdx, fillLen);
       } else {
         str[strIdx++] = fill[fillIdx++];
       }
     }
   }
 }
  private static RowKeySchema buildSchema(int[] widths) {
    RowKeySchemaBuilder builder = new RowKeySchemaBuilder(10);
    for (final int width : widths) {
      builder.addField(
          new PDatum() {
            @Override
            public boolean isNullable() {
              return false;
            }

            @Override
            public PDataType getDataType() {
              return PChar.INSTANCE;
            }

            @Override
            public Integer getMaxLength() {
              return width;
            }

            @Override
            public Integer getScale() {
              return null;
            }

            @Override
            public SortOrder getSortOrder() {
              return SortOrder.getDefault();
            }
          },
          false,
          SortOrder.getDefault());
    }
    return builder.build();
  }
Exemplo n.º 4
0
 private LiteralExpression(
     Object value, PDataType type, byte[] byteValue, Determinism determinism) {
   this(
       value,
       type,
       byteValue,
       type == null || !type.isFixedWidth() ? null : type.getMaxLength(value),
       null,
       SortOrder.getDefault(),
       determinism);
 }
Exemplo n.º 5
0
 private static int getBytesInCharNoException(byte b, SortOrder sortOrder) {
   Preconditions.checkNotNull(sortOrder);
   if (sortOrder == SortOrder.DESC) {
     b = SortOrder.invert(b);
   }
   int c = b & 0xff;
   if ((c & BYTES_1_MASK) == 0) return 1;
   if ((c & BYTES_2_MASK) == 0xC0) return 2;
   if ((c & BYTES_3_MASK) == 0xE0) return 3;
   if ((c & BYTES_4_MASK) == 0xF0) return 4;
   return -1;
 }
 @Test
 public void testParseCreateTableInlinePrimaryKeyWithOrder() throws Exception {
   for (String order : new String[] {"asc", "desc"}) {
     String s =
         "create table core.entity_history_archive (id char(15) primary key ${o})"
             .replace("${o}", order);
     CreateTableStatement stmt =
         (CreateTableStatement) new SQLParser(new StringReader(s)).parseStatement();
     List<ColumnDef> columnDefs = stmt.getColumnDefs();
     assertEquals(1, columnDefs.size());
     assertEquals(SortOrder.fromDDLValue(order), columnDefs.iterator().next().getSortOrder());
   }
 }
Exemplo n.º 7
0
 @Override
 public void write(DataOutput output) throws IOException {
   WritableUtils.writeVInt(
       output, (byteValue.length + 1) * (this.determinism == Determinism.ALWAYS ? 1 : -1));
   output.write(byteValue);
   // since we need to support clients of a lower version, serialize the determinism enum ordinal
   // in the int used to
   // serialize sort order system value (which is either 1 or 2)
   int sortOrderAndDeterminism =
       ((this.determinism.ordinal() + 1) << 2) + sortOrder.getSystemValue();
   WritableUtils.writeVInt(output, sortOrderAndDeterminism);
   WritableUtils.writeVInt(output, this.type == null ? -1 : this.type.ordinal());
 }
 @Override
 public void readFields(DataInput input) throws IOException {
   // read/write type ordinal, maxLength presence, scale presence and isNullable bit together to
   // save space
   int typeAndFlag = WritableUtils.readVInt(input);
   isNullable = (typeAndFlag & 0x01) != 0;
   if ((typeAndFlag & 0x02) != 0) {
     scale = WritableUtils.readVInt(input);
   }
   if ((typeAndFlag & 0x04) != 0) {
     maxLength = WritableUtils.readVInt(input);
   }
   type = PDataType.values()[typeAndFlag >>> 3];
   sortOrder = SortOrder.fromSystemValue(WritableUtils.readVInt(input));
 }
 @Test
 public void testParseCreateTablePrimaryKeyConstraintWithOrder() throws Exception {
   for (String order : new String[] {"asc", "desc"}) {
     String s =
         "create table core.entity_history_archive (id CHAR(15), name VARCHAR(150), constraint pk primary key (id ${o}, name ${o}))"
             .replace("${o}", order);
     CreateTableStatement stmt =
         (CreateTableStatement) new SQLParser(new StringReader(s)).parseStatement();
     PrimaryKeyConstraint pkConstraint = stmt.getPrimaryKeyConstraint();
     List<Pair<ColumnName, SortOrder>> columns = pkConstraint.getColumnNames();
     assertEquals(2, columns.size());
     for (Pair<ColumnName, SortOrder> pair : columns) {
       assertEquals(
           SortOrder.fromDDLValue(order), pkConstraint.getColumn(pair.getFirst()).getSecond());
     }
   }
 }
 @Override
 public void write(DataOutput output) throws IOException {
   // read/write type ordinal, maxLength presence, scale presence and isNullable bit together to
   // save space
   int typeAndFlag =
       (isNullable ? 1 : 0)
           | ((scale != null ? 1 : 0) << 1)
           | ((maxLength != null ? 1 : 0) << 2)
           | (type.ordinal() << 3);
   WritableUtils.writeVInt(output, typeAndFlag);
   if (scale != null) {
     WritableUtils.writeVInt(output, scale);
   }
   if (maxLength != null) {
     WritableUtils.writeVInt(output, maxLength);
   }
   WritableUtils.writeVInt(output, sortOrder.getSystemValue());
 }
Exemplo n.º 11
0
/**
 * Static class for various schema-related utilities
 *
 * @since 0.1
 */
public class SchemaUtil {
  private static final int VAR_LENGTH_ESTIMATE = 10;
  private static final int VAR_KV_LENGTH_ESTIMATE = 50;
  public static final String ESCAPE_CHARACTER = "\"";
  public static final DataBlockEncoding DEFAULT_DATA_BLOCK_ENCODING = DataBlockEncoding.FAST_DIFF;
  public static final PDatum VAR_BINARY_DATUM =
      new PDatum() {

        @Override
        public boolean isNullable() {
          return false;
        }

        @Override
        public PDataType getDataType() {
          return PVarbinary.INSTANCE;
        }

        @Override
        public Integer getMaxLength() {
          return null;
        }

        @Override
        public Integer getScale() {
          return null;
        }

        @Override
        public SortOrder getSortOrder() {
          return SortOrder.getDefault();
        }
      };
  public static final RowKeySchema VAR_BINARY_SCHEMA =
      new RowKeySchemaBuilder(1).addField(VAR_BINARY_DATUM, false, SortOrder.getDefault()).build();

  /** May not be instantiated */
  private SchemaUtil() {}

  public static boolean isPKColumn(PColumn column) {
    return column.getFamilyName() == null;
  }

  /**
   * Imperfect estimate of row size given a PTable TODO: keep row count in stats table and use total
   * size / row count instead
   *
   * @param table
   * @return estimate of size in bytes of a row
   */
  public static long estimateRowSize(PTable table) {
    int keyLength = estimateKeyLength(table);
    long rowSize = 0;
    for (PColumn column : table.getColumns()) {
      if (!SchemaUtil.isPKColumn(column)) {
        PDataType type = column.getDataType();
        Integer maxLength = column.getMaxLength();
        int valueLength =
            !type.isFixedWidth()
                ? VAR_KV_LENGTH_ESTIMATE
                : maxLength == null ? type.getByteSize() : maxLength;
        rowSize +=
            KeyValue.getKeyValueDataStructureSize(
                keyLength,
                column.getFamilyName().getBytes().length,
                column.getName().getBytes().length,
                valueLength);
      }
    }
    // Empty key value
    rowSize +=
        KeyValue.getKeyValueDataStructureSize(
            keyLength,
            getEmptyColumnFamily(table).length,
            QueryConstants.EMPTY_COLUMN_BYTES.length,
            0);
    return rowSize;
  }

  /**
   * Estimate the max key length in bytes of the PK for a given table
   *
   * @param table the table
   * @return the max PK length
   */
  public static int estimateKeyLength(PTable table) {
    int maxKeyLength = 0;
    // Calculate the max length of a key (each part must currently be of a fixed width)
    int i = 0;
    List<PColumn> columns = table.getPKColumns();
    while (i < columns.size()) {
      PColumn keyColumn = columns.get(i++);
      PDataType type = keyColumn.getDataType();
      Integer maxLength = keyColumn.getMaxLength();
      maxKeyLength +=
          !type.isFixedWidth()
              ? VAR_LENGTH_ESTIMATE
              : maxLength == null ? type.getByteSize() : maxLength;
    }
    return maxKeyLength;
  }

  /**
   * Normalize an identifier. If name is surrounded by double quotes, it is used as-is, otherwise
   * the name is upper caased.
   *
   * @param name the parsed identifier
   * @return the normalized identifier
   */
  public static String normalizeIdentifier(String name) {
    if (name == null) {
      return name;
    }
    if (isCaseSensitive(name)) {
      // Don't upper case if in quotes
      return name.substring(1, name.length() - 1);
    }
    return name.toUpperCase();
  }

  /**
   * Normalizes the fulltableName . Uses {@linkplain normalizeIdentifier}
   *
   * @param fullTableName
   * @return
   */
  public static String normalizeFullTableName(String fullTableName) {
    String schemaName = SchemaUtil.getSchemaNameFromFullName(fullTableName);
    String tableName = SchemaUtil.getTableNameFromFullName(fullTableName);
    String normalizedTableName = StringUtil.EMPTY_STRING;
    if (!schemaName.isEmpty()) {
      normalizedTableName = normalizeIdentifier(schemaName) + QueryConstants.NAME_SEPARATOR;
    }
    return normalizedTableName + normalizeIdentifier(tableName);
  }

  public static boolean isCaseSensitive(String name) {
    return name != null && name.length() > 0 && name.charAt(0) == '"';
  }

  public static <T> List<T> concat(List<T> l1, List<T> l2) {
    int size1 = l1.size();
    if (size1 == 0) {
      return l2;
    }
    int size2 = l2.size();
    if (size2 == 0) {
      return l1;
    }
    List<T> l3 = new ArrayList<T>(size1 + size2);
    l3.addAll(l1);
    l3.addAll(l2);
    return l3;
  }

  /**
   * Get the key used in the Phoenix metadata row for a table definition
   *
   * @param schemaName
   * @param tableName
   */
  public static byte[] getTableKey(byte[] tenantId, byte[] schemaName, byte[] tableName) {
    return ByteUtil.concat(
        tenantId,
        QueryConstants.SEPARATOR_BYTE_ARRAY,
        schemaName,
        QueryConstants.SEPARATOR_BYTE_ARRAY,
        tableName);
  }

  /**
   * Get the key used in the Phoenix function data row for a function definition
   *
   * @param tenantId
   * @param functionName
   */
  public static byte[] getFunctionKey(byte[] tenantId, byte[] functionName) {
    return ByteUtil.concat(tenantId, QueryConstants.SEPARATOR_BYTE_ARRAY, functionName);
  }

  public static byte[] getTableKey(String tenantId, String schemaName, String tableName) {
    return ByteUtil.concat(
        tenantId == null ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(tenantId),
        QueryConstants.SEPARATOR_BYTE_ARRAY,
        schemaName == null ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(schemaName),
        QueryConstants.SEPARATOR_BYTE_ARRAY,
        Bytes.toBytes(tableName));
  }

  public static String getTableName(String schemaName, String tableName) {
    return getName(schemaName, tableName, false);
  }

  private static String getName(String optionalQualifier, String name, boolean caseSensitive) {
    String cq = caseSensitive ? "\"" + name + "\"" : name;
    if (optionalQualifier == null || optionalQualifier.isEmpty()) {
      return cq;
    }
    String cf = caseSensitive ? "\"" + optionalQualifier + "\"" : optionalQualifier;
    return cf + QueryConstants.NAME_SEPARATOR + cq;
  }

  public static String getTableName(byte[] schemaName, byte[] tableName) {
    return getName(schemaName, tableName);
  }

  public static String getColumnDisplayName(byte[] cf, byte[] cq) {
    return getName(cf == null || cf.length == 0 ? ByteUtil.EMPTY_BYTE_ARRAY : cf, cq);
  }

  public static String getColumnDisplayName(String cf, String cq) {
    return getName(cf == null || cf.isEmpty() ? null : cf, cq, false);
  }

  public static String getCaseSensitiveColumnDisplayName(String cf, String cq) {
    return getName(cf == null || cf.isEmpty() ? null : cf, cq, true);
  }

  public static String getMetaDataEntityName(
      String schemaName, String tableName, String familyName, String columnName) {
    if ((schemaName == null || schemaName.isEmpty())
        && (tableName == null || tableName.isEmpty())) {
      if (columnName == null || columnName.isEmpty()) {
        return familyName;
      }
      return getName(familyName, columnName, false);
    }
    if ((familyName == null || familyName.isEmpty())
        && (columnName == null || columnName.isEmpty())) {
      return getName(schemaName, tableName, false);
    }
    return getName(
        getName(schemaName, tableName, false), getName(familyName, columnName, false), false);
  }

  public static String getColumnName(String familyName, String columnName) {
    return getName(familyName, columnName, false);
  }

  public static byte[] getTableNameAsBytes(String schemaName, String tableName) {
    if (schemaName == null || schemaName.length() == 0) {
      return StringUtil.toBytes(tableName);
    }
    return getTableNameAsBytes(StringUtil.toBytes(schemaName), StringUtil.toBytes(tableName));
  }

  public static byte[] getTableNameAsBytes(byte[] schemaName, byte[] tableName) {
    return getNameAsBytes(schemaName, tableName);
  }

  private static byte[] getNameAsBytes(byte[] nameOne, byte[] nameTwo) {
    if (nameOne == null || nameOne.length == 0) {
      return nameTwo;
    } else if ((nameTwo == null || nameTwo.length == 0)) {
      return nameOne;
    } else {
      return ByteUtil.concat(nameOne, QueryConstants.NAME_SEPARATOR_BYTES, nameTwo);
    }
  }

  public static String getName(byte[] nameOne, byte[] nameTwo) {
    return Bytes.toString(getNameAsBytes(nameOne, nameTwo));
  }

  public static int getVarCharLength(byte[] buf, int keyOffset, int maxLength) {
    return getVarCharLength(buf, keyOffset, maxLength, 1);
  }

  public static int getVarCharLength(byte[] buf, int keyOffset, int maxLength, int skipCount) {
    int length = 0;
    for (int i = 0; i < skipCount; i++) {
      while (length < maxLength && buf[keyOffset + length] != QueryConstants.SEPARATOR_BYTE) {
        length++;
      }
      if (i != skipCount - 1) { // skip over the separator if it's not the last one.
        length++;
      }
    }
    return length;
  }

  public static int getVarChars(byte[] rowKey, byte[][] rowKeyMetadata) {
    return getVarChars(rowKey, 0, rowKey.length, 0, rowKeyMetadata);
  }

  public static int getVarChars(byte[] rowKey, int colMetaDataLength, byte[][] colMetaData) {
    return getVarChars(rowKey, 0, rowKey.length, 0, colMetaDataLength, colMetaData);
  }

  public static int getVarChars(
      byte[] rowKey, int keyOffset, int keyLength, int colMetaDataOffset, byte[][] colMetaData) {
    return getVarChars(
        rowKey, keyOffset, keyLength, colMetaDataOffset, colMetaData.length, colMetaData);
  }

  public static int getVarChars(
      byte[] rowKey,
      int keyOffset,
      int keyLength,
      int colMetaDataOffset,
      int colMetaDataLength,
      byte[][] colMetaData) {
    int i, offset = keyOffset;
    for (i = colMetaDataOffset; i < colMetaDataLength && keyLength > 0; i++) {
      int length = getVarCharLength(rowKey, offset, keyLength);
      byte[] b = new byte[length];
      System.arraycopy(rowKey, offset, b, 0, length);
      offset += length + 1;
      keyLength -= length + 1;
      colMetaData[i] = b;
    }
    return i;
  }

  public static String findExistingColumn(PTable table, List<PColumn> columns) {
    for (PColumn column : columns) {
      PName familyName = column.getFamilyName();
      if (familyName == null) {
        try {
          return table.getPKColumn(column.getName().getString()).getName().getString();
        } catch (ColumnNotFoundException e) {
          continue;
        }
      } else {
        try {
          return table
              .getColumnFamily(familyName.getString())
              .getColumn(column.getName().getString())
              .getName()
              .getString();
        } catch (ColumnFamilyNotFoundException e) {
          continue; // Shouldn't happen
        } catch (ColumnNotFoundException e) {
          continue;
        }
      }
    }
    return null;
  }

  public static String toString(byte[][] values) {
    if (values == null) {
      return "null";
    }
    StringBuilder buf = new StringBuilder("[");
    for (byte[] value : values) {
      buf.append(Bytes.toStringBinary(value));
      buf.append(',');
    }
    buf.setCharAt(buf.length() - 1, ']');
    return buf.toString();
  }

  public static String toString(PDataType type, byte[] value) {
    boolean isString = type.isCoercibleTo(PVarchar.INSTANCE);
    return isString
        ? ("'" + type.toObject(value).toString() + "'")
        : type.toObject(value).toString();
  }

  public static byte[] getEmptyColumnFamily(
      PName defaultColumnFamily, List<PColumnFamily> families) {
    return families.isEmpty()
        ? defaultColumnFamily == null
            ? QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES
            : defaultColumnFamily.getBytes()
        : families.get(0).getName().getBytes();
  }

  public static byte[] getEmptyColumnFamily(PTable table) {
    List<PColumnFamily> families = table.getColumnFamilies();
    return families.isEmpty()
        ? table.getDefaultFamilyName() == null
            ? QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES
            : table.getDefaultFamilyName().getBytes()
        : families.get(0).getName().getBytes();
  }

  public static String getEmptyColumnFamilyAsString(PTable table) {
    List<PColumnFamily> families = table.getColumnFamilies();
    return families.isEmpty()
        ? table.getDefaultFamilyName() == null
            ? QueryConstants.DEFAULT_COLUMN_FAMILY
            : table.getDefaultFamilyName().getString()
        : families.get(0).getName().getString();
  }

  public static ImmutableBytesPtr getEmptyColumnFamilyPtr(PTable table) {
    List<PColumnFamily> families = table.getColumnFamilies();
    return families.isEmpty()
        ? table.getDefaultFamilyName() == null
            ? QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES_PTR
            : table.getDefaultFamilyName().getBytesPtr()
        : families.get(0).getName().getBytesPtr();
  }

  public static boolean isMetaTable(byte[] tableName) {
    return Bytes.compareTo(tableName, SYSTEM_CATALOG_NAME_BYTES) == 0;
  }

  public static boolean isFunctionTable(byte[] tableName) {
    return Bytes.compareTo(tableName, SYSTEM_FUNCTION_NAME_BYTES) == 0;
  }

  public static boolean isStatsTable(byte[] tableName) {
    return Bytes.compareTo(tableName, SYSTEM_STATS_NAME_BYTES) == 0;
  }

  public static boolean isSequenceTable(byte[] tableName) {
    return Bytes.compareTo(tableName, PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_NAME_BYTES) == 0;
  }

  public static boolean isSequenceTable(PTable table) {
    return PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_NAME.equals(table.getName().getString());
  }

  public static boolean isMetaTable(PTable table) {
    return PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA.equals(table.getSchemaName().getString())
        && PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE.equals(table.getTableName().getString());
  }

  public static boolean isMetaTable(byte[] schemaName, byte[] tableName) {
    return Bytes.compareTo(schemaName, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA_BYTES) == 0
        && Bytes.compareTo(tableName, PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE_BYTES) == 0;
  }

  public static boolean isMetaTable(String schemaName, String tableName) {
    return PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA.equals(schemaName)
        && PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE.equals(tableName);
  }

  public static boolean isSystemTable(byte[] fullTableName) {
    String schemaName = SchemaUtil.getSchemaNameFromFullName(fullTableName);
    if (QueryConstants.SYSTEM_SCHEMA_NAME.equals(schemaName)) return true;
    return false;
  }

  // Given the splits and the rowKeySchema, find out the keys that
  public static byte[][] processSplits(
      byte[][] splits,
      LinkedHashSet<PColumn> pkColumns,
      Integer saltBucketNum,
      boolean defaultRowKeyOrder)
      throws SQLException {
    // FIXME: shouldn't this return if splits.length == 0?
    if (splits == null) return null;
    // We do not accept user specified splits if the table is salted and we specify
    // defaultRowKeyOrder. In this case,
    // throw an exception.
    if (splits.length > 0 && saltBucketNum != null && defaultRowKeyOrder) {
      throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_SPLITS_ON_SALTED_TABLE)
          .build()
          .buildException();
    }
    // If the splits are not specified and table is salted, pre-split the table.
    if (splits.length == 0 && saltBucketNum != null) {
      splits = SaltingUtil.getSalteByteSplitPoints(saltBucketNum);
    }
    byte[][] newSplits = new byte[splits.length][];
    for (int i = 0; i < splits.length; i++) {
      newSplits[i] = processSplit(splits[i], pkColumns);
    }
    return newSplits;
  }

  // Go through each slot in the schema and try match it with the split byte array. If the split
  // does not confer to the schema, extends its length to match the schema.
  private static byte[] processSplit(byte[] split, LinkedHashSet<PColumn> pkColumns) {
    int pos = 0, offset = 0, maxOffset = split.length;
    Iterator<PColumn> iterator = pkColumns.iterator();
    while (pos < pkColumns.size()) {
      PColumn column = iterator.next();
      if (column.getDataType().isFixedWidth()) { // Fixed width
        int length = SchemaUtil.getFixedByteSize(column);
        if (maxOffset - offset < length) {
          // The split truncates the field. Fill in the rest of the part and any fields that
          // are missing after this field.
          int fillInLength = length - (maxOffset - offset);
          fillInLength += estimatePartLength(pos + 1, iterator);
          return ByteUtil.fillKey(split, split.length + fillInLength);
        }
        // Account for this field, move to next position;
        offset += length;
        pos++;
      } else { // Variable length
        // If we are the last slot, then we are done. Nothing needs to be filled in.
        if (pos == pkColumns.size() - 1) {
          break;
        }
        while (offset < maxOffset && split[offset] != QueryConstants.SEPARATOR_BYTE) {
          offset++;
        }
        if (offset == maxOffset) {
          // The var-length field does not end with a separator and it's not the last field.
          int fillInLength = 1; // SEPARATOR byte for the current var-length slot.
          fillInLength += estimatePartLength(pos + 1, iterator);
          return ByteUtil.fillKey(split, split.length + fillInLength);
        }
        // Move to the next position;
        offset += 1; // skip separator;
        pos++;
      }
    }
    return split;
  }

  // Estimate the key length after pos slot for schema.
  private static int estimatePartLength(int pos, Iterator<PColumn> iterator) {
    int length = 0;
    while (iterator.hasNext()) {
      PColumn column = iterator.next();
      if (column.getDataType().isFixedWidth()) {
        length += SchemaUtil.getFixedByteSize(column);
      } else {
        length += 1; // SEPARATOR byte.
      }
    }
    return length;
  }

  public static String getEscapedTableName(String schemaName, String tableName) {
    if (schemaName == null || schemaName.length() == 0) {
      return "\"" + tableName + "\"";
    }
    return "\"" + schemaName + "\"." + "\"" + tableName + "\"";
  }

  protected static PhoenixConnection addMetaDataColumn(
      PhoenixConnection conn, long scn, String columnDef) throws SQLException {
    PhoenixConnection metaConnection = null;
    Statement stmt = null;
    try {
      metaConnection = new PhoenixConnection(conn.getQueryServices(), conn, scn);
      try {
        stmt = metaConnection.createStatement();
        stmt.executeUpdate("ALTER TABLE SYSTEM.\"TABLE\" ADD IF NOT EXISTS " + columnDef);
        return metaConnection;
      } finally {
        if (stmt != null) {
          stmt.close();
        }
      }
    } finally {
      if (metaConnection != null) {
        metaConnection.close();
      }
    }
  }

  public static boolean columnExists(PTable table, String columnName) {
    try {
      table.getColumn(columnName);
      return true;
    } catch (ColumnNotFoundException e) {
      return false;
    } catch (AmbiguousColumnException e) {
      return true;
    }
  }

  public static String getSchemaNameFromFullName(String tableName) {
    int index = tableName.indexOf(QueryConstants.NAME_SEPARATOR);
    if (index < 0) {
      return StringUtil.EMPTY_STRING;
    }
    return tableName.substring(0, index);
  }

  private static int indexOf(byte[] bytes, byte b) {
    for (int i = 0; i < bytes.length; i++) {
      if (bytes[i] == b) {
        return i;
      }
    }
    return -1;
  }

  public static String getSchemaNameFromFullName(byte[] tableName) {
    if (tableName == null) {
      return null;
    }
    int index = indexOf(tableName, QueryConstants.NAME_SEPARATOR_BYTE);
    if (index < 0) {
      return StringUtil.EMPTY_STRING;
    }
    return Bytes.toString(tableName, 0, index);
  }

  public static String getTableNameFromFullName(byte[] tableName) {
    if (tableName == null) {
      return null;
    }
    int index = indexOf(tableName, QueryConstants.NAME_SEPARATOR_BYTE);
    if (index < 0) {
      return Bytes.toString(tableName);
    }
    return Bytes.toString(tableName, index + 1, tableName.length - index - 1);
  }

  public static String getTableNameFromFullName(String tableName) {
    int index = tableName.indexOf(QueryConstants.NAME_SEPARATOR);
    if (index < 0) {
      return tableName;
    }
    return tableName.substring(index + 1, tableName.length());
  }

  public static byte[] getTableKeyFromFullName(String fullTableName) {
    int index = fullTableName.indexOf(QueryConstants.NAME_SEPARATOR);
    if (index < 0) {
      return getTableKey(null, null, fullTableName);
    }
    String schemaName = fullTableName.substring(0, index);
    String tableName = fullTableName.substring(index + 1);
    return getTableKey(null, schemaName, tableName);
  }

  private static int getTerminatorCount(RowKeySchema schema) {
    int nTerminators = 0;
    for (int i = 0; i < schema.getFieldCount(); i++) {
      Field field = schema.getField(i);
      // We won't have a terminator on the last PK column
      // unless it is variable length and exclusive, but
      // having the extra byte irregardless won't hurt anything
      if (!field.getDataType().isFixedWidth()) {
        nTerminators++;
      }
    }
    return nTerminators;
  }

  public static int getMaxKeyLength(RowKeySchema schema, List<List<KeyRange>> slots) {
    int maxKeyLength = getTerminatorCount(schema) * 2;
    for (List<KeyRange> slot : slots) {
      int maxSlotLength = 0;
      for (KeyRange range : slot) {
        int maxRangeLength = Math.max(range.getLowerRange().length, range.getUpperRange().length);
        if (maxSlotLength < maxRangeLength) {
          maxSlotLength = maxRangeLength;
        }
      }
      maxKeyLength += maxSlotLength;
    }
    return maxKeyLength;
  }

  public static int getFixedByteSize(PDatum e) {
    assert (e.getDataType().isFixedWidth());
    Integer maxLength = e.getMaxLength();
    return maxLength == null ? e.getDataType().getByteSize() : maxLength;
  }

  public static short getMaxKeySeq(PTable table) {
    int offset = 0;
    if (table.getBucketNum() != null) {
      offset++;
    }
    // TODO: for tenant-specific table on tenant-specific connection,
    // we should subtract one for tenant column and another one for
    // index ID
    return (short) (table.getPKColumns().size() - offset);
  }

  public static int getPKPosition(PTable table, PColumn column) {
    // TODO: when PColumn has getPKPosition, use that instead
    return table.getPKColumns().indexOf(column);
  }

  public static String getEscapedFullColumnName(String fullColumnName) {
    if (fullColumnName.startsWith(ESCAPE_CHARACTER)) {
      return fullColumnName;
    }
    int index = fullColumnName.indexOf(QueryConstants.NAME_SEPARATOR);
    if (index < 0) {
      return getEscapedArgument(fullColumnName);
    }
    String columnFamily = fullColumnName.substring(0, index);
    String columnName = fullColumnName.substring(index + 1);
    return getEscapedArgument(columnFamily)
        + QueryConstants.NAME_SEPARATOR
        + getEscapedArgument(columnName);
  }

  public static String getEscapedFullTableName(String fullTableName) {
    final String schemaName = getSchemaNameFromFullName(fullTableName);
    final String tableName = getTableNameFromFullName(fullTableName);
    return getEscapedTableName(schemaName, tableName);
  }

  /**
   * Escapes the given argument with {@value #ESCAPE_CHARACTER}
   *
   * @param argument any non null value.
   * @return
   */
  public static String getEscapedArgument(String argument) {
    Preconditions.checkNotNull(argument, "Argument passed cannot be null");
    return ESCAPE_CHARACTER + argument + ESCAPE_CHARACTER;
  }

  /**
   * @return a fully qualified column name in the format: "CFNAME"."COLNAME" or "COLNAME" depending
   *     on whether or not there is a column family name present.
   */
  public static String getQuotedFullColumnName(PColumn pCol) {
    checkNotNull(pCol);
    String columnName = pCol.getName().getString();
    String columnFamilyName =
        pCol.getFamilyName() != null ? pCol.getFamilyName().getString() : null;
    return getQuotedFullColumnName(columnFamilyName, columnName);
  }

  /**
   * @return a fully qualified column name in the format: "CFNAME"."COLNAME" or "COLNAME" depending
   *     on whether or not there is a column family name present.
   */
  public static String getQuotedFullColumnName(
      @Nullable String columnFamilyName, String columnName) {
    checkArgument(!isNullOrEmpty(columnName), "Column name cannot be null or empty");
    return columnFamilyName == null
        ? ("\"" + columnName + "\"")
        : ("\""
            + columnFamilyName
            + "\""
            + QueryConstants.NAME_SEPARATOR
            + "\""
            + columnName
            + "\"");
  }

  public static boolean hasHTableDescriptorProps(Map<String, Object> tableProps) {
    int pTablePropCount = 0;
    for (String prop : tableProps.keySet()) {
      if (TableProperty.isPhoenixTableProperty(prop)) {
        pTablePropCount++;
      }
    }
    return tableProps.size() - pTablePropCount > 0;
  }

  /**
   * Replaces all occurrences of {@link #ESCAPE_CHARACTER} with an empty character.
   *
   * @param fullColumnName
   * @return
   */
  public static String getUnEscapedFullColumnName(String fullColumnName) {
    checkArgument(!isNullOrEmpty(fullColumnName), "Column name cannot be null or empty");
    fullColumnName = fullColumnName.replaceAll(ESCAPE_CHARACTER, "");
    return fullColumnName.trim();
  }

  /**
   * Return the separator byte to use based on:
   *
   * @param rowKeyOrderOptimizable whether or not the table may optimize descending row keys. If the
   *     table has no descending row keys, this will be true. Also, if the table has been upgraded
   *     (using a new -u option for psql.py), then it'll be true
   * @param isNullValue whether or not the value is null. We use a null byte still if the value is
   *     null regardless of sort order since nulls will always sort first this way.
   * @param sortOrder whether the value sorts ascending or descending.
   * @return the byte to use as the separator
   */
  public static byte getSeparatorByte(
      boolean rowKeyOrderOptimizable, boolean isNullValue, SortOrder sortOrder) {
    return !rowKeyOrderOptimizable || isNullValue || sortOrder == SortOrder.ASC
        ? QueryConstants.SEPARATOR_BYTE
        : QueryConstants.DESC_SEPARATOR_BYTE;
  }

  public static byte getSeparatorByte(
      boolean rowKeyOrderOptimizable, boolean isNullValue, Field f) {
    return getSeparatorByte(rowKeyOrderOptimizable, isNullValue, f.getSortOrder());
  }

  public static byte getSeparatorByte(
      boolean rowKeyOrderOptimizable, boolean isNullValue, Expression e) {
    return getSeparatorByte(rowKeyOrderOptimizable, isNullValue, e.getSortOrder());
  }

  /**
   * Get list of ColumnInfos that contain Column Name and its associated PDataType for an import.
   * The supplied list of columns can be null -- if it is non-null, it represents a user-supplied
   * list of columns to be imported.
   *
   * @param conn Phoenix connection from which metadata will be read
   * @param tableName Phoenix table name whose columns are to be checked. Can include a schema name
   * @param columns user-supplied list of import columns, can be null
   * @param strict if true, an exception will be thrown if unknown columns are supplied
   */
  public static List<ColumnInfo> generateColumnInfo(
      Connection conn, String tableName, List<String> columns, boolean strict) throws SQLException {
    Map<String, Integer> columnNameToTypeMap = Maps.newLinkedHashMap();
    Set<String> ambiguousColumnNames = new HashSet<String>();
    Map<String, Integer> fullColumnNameToTypeMap = Maps.newLinkedHashMap();
    DatabaseMetaData dbmd = conn.getMetaData();
    int unfoundColumnCount = 0;
    // TODO: escape wildcard characters here because we don't want that
    // behavior here
    String escapedTableName = StringUtil.escapeLike(tableName);
    String[] schemaAndTable = escapedTableName.split("\\.");
    ResultSet rs = null;
    try {
      rs =
          dbmd.getColumns(
              null,
              (schemaAndTable.length == 1 ? "" : schemaAndTable[0]),
              (schemaAndTable.length == 1 ? escapedTableName : schemaAndTable[1]),
              null);
      while (rs.next()) {
        String colName = rs.getString(QueryUtil.COLUMN_NAME_POSITION);
        String colFam = rs.getString(QueryUtil.COLUMN_FAMILY_POSITION);

        // use family qualifier, if available, otherwise, use column name
        String fullColumn = (colFam == null ? colName : String.format("%s.%s", colFam, colName));
        String sqlTypeName = rs.getString(QueryUtil.DATA_TYPE_NAME_POSITION);

        // allow for both bare and family qualified names.
        if (columnNameToTypeMap.keySet().contains(colName)) {
          ambiguousColumnNames.add(colName);
        }
        columnNameToTypeMap.put(colName, PDataType.fromSqlTypeName(sqlTypeName).getSqlType());
        fullColumnNameToTypeMap.put(
            fullColumn, PDataType.fromSqlTypeName(sqlTypeName).getSqlType());
      }
      if (columnNameToTypeMap.isEmpty()) {
        throw new IllegalArgumentException("Table " + tableName + " not found");
      }
    } finally {
      if (rs != null) {
        rs.close();
      }
    }
    List<ColumnInfo> columnInfoList = Lists.newArrayList();
    Set<String> unresolvedColumnNames = new TreeSet<String>();
    if (columns == null) {
      // use family qualified names by default, if no columns are specified.
      for (Map.Entry<String, Integer> entry : fullColumnNameToTypeMap.entrySet()) {
        columnInfoList.add(new ColumnInfo(entry.getKey(), entry.getValue()));
      }
    } else {
      // Leave "null" as indication to skip b/c it doesn't exist
      for (int i = 0; i < columns.size(); i++) {
        String columnName = columns.get(i).trim();
        Integer sqlType = null;
        if (fullColumnNameToTypeMap.containsKey(columnName)) {
          sqlType = fullColumnNameToTypeMap.get(columnName);
        } else if (columnNameToTypeMap.containsKey(columnName)) {
          if (ambiguousColumnNames.contains(columnName)) {
            unresolvedColumnNames.add(columnName);
          }
          // fall back to bare column name.
          sqlType = columnNameToTypeMap.get(columnName);
        }
        if (unresolvedColumnNames.size() > 0) {
          StringBuilder exceptionMessage = new StringBuilder();
          boolean first = true;
          exceptionMessage.append(
              "Unable to resolve these column names to a single column family:\n");
          for (String col : unresolvedColumnNames) {
            if (first) first = false;
            else exceptionMessage.append(",");
            exceptionMessage.append(col);
          }
          exceptionMessage.append("\nAvailable columns with column families:\n");
          first = true;
          for (String col : fullColumnNameToTypeMap.keySet()) {
            if (first) first = false;
            else exceptionMessage.append(",");
            exceptionMessage.append(col);
          }
          throw new SQLException(exceptionMessage.toString());
        }

        if (sqlType == null) {
          if (strict) {
            throw new SQLExceptionInfo.Builder(SQLExceptionCode.COLUMN_NOT_FOUND)
                .setColumnName(columnName)
                .setTableName(tableName)
                .build()
                .buildException();
          }
          unfoundColumnCount++;
        } else {
          columnInfoList.add(new ColumnInfo(columnName, sqlType));
        }
      }
      if (unfoundColumnCount == columns.size()) {
        throw new SQLExceptionInfo.Builder(SQLExceptionCode.COLUMN_NOT_FOUND)
            .setColumnName(Arrays.toString(columns.toArray(new String[0])))
            .setTableName(tableName)
            .build()
            .buildException();
      }
    }
    return columnInfoList;
  }

  public static boolean hasRowTimestampColumn(PTable table) {
    return table.getRowTimestampColPos() > 0;
  }

  public static byte[] getTableKey(PTable dataTable) {
    PName tenantId = dataTable.getTenantId();
    PName schemaName = dataTable.getSchemaName();
    return getTableKey(
        tenantId == null ? ByteUtil.EMPTY_BYTE_ARRAY : tenantId.getBytes(),
        schemaName == null ? ByteUtil.EMPTY_BYTE_ARRAY : schemaName.getBytes(),
        dataTable.getTableName().getBytes());
  }
}
Exemplo n.º 12
0
 @Override
 public SortOrder getSortOrder() {
   return SortOrder.getDefault();
 }
Exemplo n.º 13
0
public class StringUtil {
  public static final String EMPTY_STRING = "";
  // Masks to determine how many bytes are in each character
  // From http://tools.ietf.org/html/rfc3629#section-3
  public static final byte SPACE_UTF8 = 0x20;
  private static final int BYTES_1_MASK = 0xFF << 7; // 0xxxxxxx is a single byte char
  private static final int BYTES_2_MASK = 0xFF << 5; // 110xxxxx is a double byte char
  private static final int BYTES_3_MASK = 0xFF << 4; // 1110xxxx is a triple byte char
  private static final int BYTES_4_MASK = 0xFF << 3; // 11110xxx is a quadruple byte char

  public static final byte INVERTED_SPACE_UTF8 =
      SortOrder.invert(new byte[] {SPACE_UTF8}, 0, new byte[1], 0, 1)[0];
  public static final char SINGLE_CHAR_WILDCARD = '?';
  public static final char SINGLE_CHAR_LIKE = '_';
  public static final char MULTI_CHAR_WILDCARD = '*';
  public static final char MULTI_CHAR_LIKE = '%';
  public static final String[] LIKE_ESCAPE_SEQS =
      new String[] {"\\" + SINGLE_CHAR_LIKE, "\\" + MULTI_CHAR_LIKE};
  public static final String[] LIKE_UNESCAPED_SEQS =
      new String[] {"" + SINGLE_CHAR_LIKE, "" + MULTI_CHAR_LIKE};

  private StringUtil() {}

  /** Replace instances of character ch in String value with String replacement */
  public static String replaceChar(String value, char ch, CharSequence replacement) {
    if (value == null) return null;
    int i = value.indexOf(ch);
    if (i == -1) return value; // nothing to do

    // we've got at least one character to replace
    StringBuilder buf = new StringBuilder(value.length() + 16); // some extra space
    int j = 0;
    while (i != -1) {
      buf.append(value, j, i).append(replacement);
      j = i + 1;
      i = value.indexOf(ch, j);
    }
    if (j < value.length()) buf.append(value, j, value.length());
    return buf.toString();
  }

  /**
   * @return the replacement of all occurrences of src[i] with target[i] in s. Src and target are
   *     not regex's so this uses simple searching with indexOf()
   */
  public static String replace(String s, String[] src, String[] target) {
    assert src != null && target != null && src.length > 0 && src.length == target.length;
    if (src.length == 1 && src[0].length() == 1) {
      return replaceChar(s, src[0].charAt(0), target[0]);
    }
    if (s == null) return null;
    StringBuilder sb = new StringBuilder(s.length());
    int pos = 0;
    int limit = s.length();
    int lastMatch = 0;
    while (pos < limit) {
      boolean matched = false;
      for (int i = 0; i < src.length; i++) {
        if (s.startsWith(src[i], pos) && src[i].length() > 0) {
          // we found a matching pattern - append the acculumation plus the replacement
          sb.append(s.substring(lastMatch, pos)).append(target[i]);
          pos += src[i].length();
          lastMatch = pos;
          matched = true;
          break;
        }
      }
      if (!matched) {
        // we didn't match any patterns, so move forward 1 character
        pos++;
      }
    }
    // see if we found any matches
    if (lastMatch == 0) {
      // we didn't match anything, so return the source string
      return s;
    }

    // apppend the trailing portion
    sb.append(s.substring(lastMatch));

    return sb.toString();
  }

  public static int getBytesInChar(byte b, SortOrder sortOrder) {
    int ret = getBytesInCharNoException(b, sortOrder);
    if (ret == -1) throw new UndecodableByteException(b);
    return ret;
  }

  private static int getBytesInCharNoException(byte b, SortOrder sortOrder) {
    Preconditions.checkNotNull(sortOrder);
    if (sortOrder == SortOrder.DESC) {
      b = SortOrder.invert(b);
    }
    int c = b & 0xff;
    if ((c & BYTES_1_MASK) == 0) return 1;
    if ((c & BYTES_2_MASK) == 0xC0) return 2;
    if ((c & BYTES_3_MASK) == 0xE0) return 3;
    if ((c & BYTES_4_MASK) == 0xF0) return 4;
    return -1;
  }

  public static int calculateUTF8Length(byte[] bytes, int offset, int length, SortOrder sortOrder) {
    int i = offset, endOffset = offset + length;
    length = 0;
    while (i < endOffset) {
      int charLength = getBytesInChar(bytes[i], sortOrder);
      i += charLength;
      length++;
    }
    return length;
  }

  // given an array of bytes containing utf-8 encoded strings, starting from curPos, ending before
  // range, and return the next character offset, -1 if no next character available or
  // UndecodableByteException
  private static int calculateNextCharOffset(
      byte[] bytes, int curPos, int range, SortOrder sortOrder) {
    int ret = getBytesInCharNoException(bytes[curPos], sortOrder);
    if (ret == -1) return -1;
    ret += curPos;
    if (ret >= range) return -1;
    return ret;
  }

  // given an array of bytes containing utf-8 encoded strings, starting from offset, and return
  // the previous character offset , -1 if UndecodableByteException. curPos points to current
  // character starting offset.
  private static int calculatePreCharOffset(
      byte[] bytes, int curPos, int offset, SortOrder sortOrder) {
    --curPos;
    for (int i = 1, pos = curPos - i + 1; i <= 4 && offset <= pos; ++i, --pos) {
      int ret = getBytesInCharNoException(bytes[pos], sortOrder);
      if (ret == i) return pos;
    }
    return -1;
  }

  // return actural offsetInBytes corresponding to offsetInStr in utf-8 encoded strings bytes
  // containing
  // @param bytes an array of bytes containing utf-8 encoded strings
  // @param offset
  // @param length
  // @param sortOrder
  // @param offsetInStr offset for utf-8 encoded strings bytes array containing. Can be negative
  // meaning counting from the end of encoded strings
  // @return actural offsetInBytes corresponding to offsetInStr. -1 if offsetInStr is out of index
  public static int calculateUTF8Offset(
      byte[] bytes, int offset, int length, SortOrder sortOrder, int offsetInStr) {
    if (offsetInStr == 0) return offset;
    int ret, range = offset + length;
    if (offsetInStr > 0) {
      ret = offset;
      while (offsetInStr > 0) {
        ret = calculateNextCharOffset(bytes, ret, range, sortOrder);
        if (ret == -1) return -1;
        --offsetInStr;
      }
    } else {
      ret = offset + length;
      while (offsetInStr < 0) {
        ret = calculatePreCharOffset(bytes, ret, offset, sortOrder);
        // if calculateCurCharOffset returns -1, ret must be smaller than offset
        if (ret < offset) return -1;
        ++offsetInStr;
      }
    }
    return ret;
  }

  // Given an array of bytes containing encoding utf-8 encoded strings, the offset and a length
  // parameter, return the actual index into the byte array which would represent a substring
  // of <length> starting from the character at <offset>. We assume the <offset> is the start
  // byte of an UTF-8 character.
  public static int getByteLengthForUtf8SubStr(
      byte[] bytes, int offset, int length, SortOrder sortOrder) {
    int byteLength = 0;
    while (length > 0 && offset + byteLength < bytes.length) {
      int charLength = getBytesInChar(bytes[offset + byteLength], sortOrder);
      byteLength += charLength;
      length--;
    }
    return byteLength;
  }

  public static boolean hasMultiByteChars(String s) {
    for (int i = 0; i < s.length(); i++) {
      char c = s.charAt(i);
      if (c > 0x007F) {
        return true;
      }
    }
    return false;
  }

  public static int getFirstNonBlankCharIdxFromStart(
      byte[] string, int offset, int length, SortOrder sortOrder) {
    int i = offset;
    byte space = sortOrder == SortOrder.ASC ? SPACE_UTF8 : INVERTED_SPACE_UTF8;
    for (; i < offset + length; i++) {
      if (string[i] != space) {
        break;
      }
    }
    return i;
  }

  public static int getFirstNonBlankCharIdxFromEnd(
      byte[] string, int offset, int length, SortOrder sortOrder) {
    int i = offset + length - 1;
    byte space = sortOrder == SortOrder.ASC ? SPACE_UTF8 : INVERTED_SPACE_UTF8;
    for (; i >= offset; i--) {
      if (string[i] != space) {
        break;
      }
    }
    return i;
  }

  // A toBytes function backed up HBase's utility function, but would accept null input, in which
  // case it returns an empty byte array.
  public static byte[] toBytes(String input) {
    if (input == null) {
      return ByteUtil.EMPTY_BYTE_ARRAY;
    }
    return Bytes.toBytes(input);
  }

  public static String escapeLike(String s) {
    return replace(s, LIKE_UNESCAPED_SEQS, LIKE_ESCAPE_SEQS);
  }

  public static int getUnpaddedCharLength(byte[] b, int offset, int length, SortOrder sortOrder) {
    return getFirstNonBlankCharIdxFromEnd(b, offset, length, sortOrder) - offset + 1;
  }

  public static byte[] padChar(byte[] value, int offset, int length, int paddedLength) {
    byte[] key = new byte[paddedLength];
    System.arraycopy(value, offset, key, 0, length);
    Arrays.fill(key, length, paddedLength, SPACE_UTF8);
    return key;
  }

  public static byte[] padChar(byte[] value, Integer byteSize) {
    byte[] newValue = Arrays.copyOf(value, byteSize);
    if (newValue.length > value.length) {
      Arrays.fill(newValue, value.length, newValue.length, SPACE_UTF8);
    }
    return newValue;
  }

  /**
   * Lame - StringBuilder.equals is retarded.
   *
   * @param b1
   * @param b2
   * @return whether or not the two builders consist the same sequence of characters
   */
  public static boolean equals(StringBuilder b1, StringBuilder b2) {
    if (b1.length() != b2.length()) {
      return false;
    }
    for (int i = 0; i < b1.length(); i++) {
      if (b1.charAt(i) != b2.charAt(i)) {
        return false;
      }
    }
    return true;
  }

  /**
   * LPAD implementation
   *
   * @param str array containing string to be left padded
   * @param strOffset byte offset of string
   * @param strLength byte length of string
   * @param fill array containing fill values
   * @param fillOffset byte offset of fill
   * @param fillLength byte length of fill
   * @param invertFill if true inverts the bits in fill before filling the array
   * @param strWithPaddingLen length of the string that is returned with fill values left padded
   * @return byte[] containing left padded string
   */
  public static byte[] lpad(
      byte[] str,
      int strOffset,
      int strLength,
      byte[] fill,
      int fillOffset,
      int fillLength,
      boolean invertFill,
      int strWithPaddingLen) {
    byte[] paddedStr = new byte[strWithPaddingLen];
    int fillStopIdx = strWithPaddingLen - strLength;
    // copy fill into the start of paddedStr
    fill(paddedStr, 0, fillStopIdx, fill, fillOffset, fillOffset + fillLength, invertFill);
    // fill remaining characters with original string
    System.arraycopy(str, strOffset, paddedStr, fillStopIdx, strLength);
    return paddedStr;
  }

  /**
   * Assigns the specified byte values to elements of the specified range of the specified array of
   * bytes. The range to be filled extends from index fromIndex, inclusive, to index toIndex,
   * exclusive. (If fromIndex==toIndex, the range to be filled is empty.)
   *
   * @param str the array to be filled
   * @param strFromIdx the index of the first element (inclusive) to be filled with the fill values
   * @param strToIdx the index of the last element (exclusive) to be filled with the fill values
   * @param fillArray the values to be stored in all elements of the array
   * @param fillFromIdx the index of the first element (inclusive) to be used as fill values
   * @param filToIdx the index of the last element (exclusive) to be used as fill value
   * @param invertFill if true inverts the bits in fill before filling the array
   */
  public static void fill(
      byte[] str,
      int strFromIdx,
      int strToIdx,
      byte[] fillArray,
      int fillFromIdx,
      int fillToIdx,
      boolean invertFill) {
    rangeCheck(str.length, strFromIdx, strToIdx);
    rangeCheck(fillArray.length, fillFromIdx, fillToIdx);
    int strIdx = strFromIdx;
    byte[] fill = fillArray;
    int fillLen = fillToIdx - fillFromIdx;
    if (invertFill) fill = SortOrder.invert(fillArray, fillFromIdx, fillLen);
    while (strIdx < strToIdx) {
      int fillIdx = fillFromIdx;
      while (fillIdx < fillToIdx && strIdx < strToIdx) {
        if (strIdx + fillLen < fillToIdx) {
          System.arraycopy(fill, fillFromIdx, str, strIdx, fillLen);
        } else {
          str[strIdx++] = fill[fillIdx++];
        }
      }
    }
  }

  /**
   * Checks that fromIndex and toIndex are in the range and throws an appropriate exception, if they
   * are not
   */
  private static void rangeCheck(int length, int fromIndex, int toIndex) {
    if (fromIndex > toIndex) {
      throw new IllegalArgumentException("fromIndex(" + fromIndex + ") > toIndex(" + toIndex + ")");
    }
    if (fromIndex < 0) {
      throw new ArrayIndexOutOfBoundsException(fromIndex);
    }
    if (toIndex > length) {
      throw new ArrayIndexOutOfBoundsException(toIndex);
    }
  }

  public static String escapeStringConstant(String pattern) {
    return StringEscapeUtils.escapeSql(pattern); // Need to escape double quotes
  }

  public static String escapeBackslash(String input) {
    // see
    // http://stackoverflow.com/questions/4653831/regex-how-to-escape-backslashes-and-special-characters
    return input.replaceAll("\\\\", "\\\\\\\\");
  }
}
Exemplo n.º 14
0
 public static LiteralExpression newConstant(
     Object value, PDataType type, Integer maxLength, Integer scale, Determinism determinism)
     throws SQLException { // remove?
   return newConstant(value, type, maxLength, scale, SortOrder.getDefault(), determinism);
 }
Exemplo n.º 15
0
 public static LiteralExpression newConstant(Object value, PDataType type, Determinism determinism)
     throws SQLException {
   return newConstant(value, type, SortOrder.getDefault(), determinism);
 }
Exemplo n.º 16
0
 private static MutationState upsertSelect(
     PhoenixStatement statement,
     TableRef tableRef,
     RowProjector projector,
     ResultIterator iterator,
     int[] columnIndexes,
     int[] pkSlotIndexes)
     throws SQLException {
   try {
     PhoenixConnection connection = statement.getConnection();
     ConnectionQueryServices services = connection.getQueryServices();
     int maxSize =
         services
             .getProps()
             .getInt(
                 QueryServices.MAX_MUTATION_SIZE_ATTRIB,
                 QueryServicesOptions.DEFAULT_MAX_MUTATION_SIZE);
     int batchSize = Math.min(connection.getMutateBatchSize(), maxSize);
     boolean isAutoCommit = connection.getAutoCommit();
     byte[][] values = new byte[columnIndexes.length][];
     int rowCount = 0;
     Map<ImmutableBytesPtr, Map<PColumn, byte[]>> mutation =
         Maps.newHashMapWithExpectedSize(batchSize);
     PTable table = tableRef.getTable();
     ResultSet rs = new PhoenixResultSet(iterator, projector, statement);
     ImmutableBytesWritable ptr = new ImmutableBytesWritable();
     while (rs.next()) {
       for (int i = 0; i < values.length; i++) {
         PColumn column = table.getColumns().get(columnIndexes[i]);
         byte[] bytes = rs.getBytes(i + 1);
         ptr.set(bytes == null ? ByteUtil.EMPTY_BYTE_ARRAY : bytes);
         Object value = rs.getObject(i + 1);
         int rsPrecision = rs.getMetaData().getPrecision(i + 1);
         Integer precision = rsPrecision == 0 ? null : rsPrecision;
         int rsScale = rs.getMetaData().getScale(i + 1);
         Integer scale = rsScale == 0 ? null : rsScale;
         // We are guaranteed that the two column will have compatible types,
         // as we checked that before.
         if (!column
             .getDataType()
             .isSizeCompatible(
                 ptr,
                 value,
                 column.getDataType(),
                 precision,
                 scale,
                 column.getMaxLength(),
                 column.getScale())) {
           throw new SQLExceptionInfo.Builder(SQLExceptionCode.DATA_EXCEEDS_MAX_CAPACITY)
               .setColumnName(column.getName().getString())
               .setMessage("value=" + column.getDataType().toStringLiteral(ptr, null))
               .build()
               .buildException();
         }
         column
             .getDataType()
             .coerceBytes(
                 ptr,
                 value,
                 column.getDataType(),
                 precision,
                 scale,
                 SortOrder.getDefault(),
                 column.getMaxLength(),
                 column.getScale(),
                 column.getSortOrder());
         values[i] = ByteUtil.copyKeyBytesIfNecessary(ptr);
       }
       setValues(values, pkSlotIndexes, columnIndexes, table, mutation);
       rowCount++;
       // Commit a batch if auto commit is true and we're at our batch size
       if (isAutoCommit && rowCount % batchSize == 0) {
         MutationState state = new MutationState(tableRef, mutation, 0, maxSize, connection);
         connection.getMutationState().join(state);
         connection.commit();
         mutation.clear();
       }
     }
     // If auto commit is true, this last batch will be committed upon return
     return new MutationState(
         tableRef, mutation, rowCount / batchSize * batchSize, maxSize, connection);
   } finally {
     iterator.close();
   }
 }
  @Override
  protected RegionScanner doPostScannerOpen(
      final ObserverContext<RegionCoprocessorEnvironment> c, final Scan scan, final RegionScanner s)
      throws IOException {
    byte[] isUngroupedAgg = scan.getAttribute(BaseScannerRegionObserver.UNGROUPED_AGG);
    if (isUngroupedAgg == null) {
      return s;
    }

    final ScanProjector p = ScanProjector.deserializeProjectorFromScan(scan);
    final HashJoinInfo j = HashJoinInfo.deserializeHashJoinFromScan(scan);
    RegionScanner theScanner = s;
    if (p != null || j != null) {
      theScanner =
          new HashJoinRegionScanner(s, p, j, ScanUtil.getTenantId(scan), c.getEnvironment());
    }
    final RegionScanner innerScanner = theScanner;

    byte[] indexUUID = scan.getAttribute(PhoenixIndexCodec.INDEX_UUID);
    PTable projectedTable = null;
    List<Expression> selectExpressions = null;
    byte[] upsertSelectTable = scan.getAttribute(BaseScannerRegionObserver.UPSERT_SELECT_TABLE);
    boolean isUpsert = false;
    boolean isDelete = false;
    byte[] deleteCQ = null;
    byte[] deleteCF = null;
    byte[][] values = null;
    byte[] emptyCF = null;
    ImmutableBytesWritable ptr = null;
    if (upsertSelectTable != null) {
      isUpsert = true;
      projectedTable = deserializeTable(upsertSelectTable);
      selectExpressions =
          deserializeExpressions(scan.getAttribute(BaseScannerRegionObserver.UPSERT_SELECT_EXPRS));
      values = new byte[projectedTable.getPKColumns().size()][];
      ptr = new ImmutableBytesWritable();
    } else {
      byte[] isDeleteAgg = scan.getAttribute(BaseScannerRegionObserver.DELETE_AGG);
      isDelete = isDeleteAgg != null && Bytes.compareTo(PDataType.TRUE_BYTES, isDeleteAgg) == 0;
      if (!isDelete) {
        deleteCF = scan.getAttribute(BaseScannerRegionObserver.DELETE_CF);
        deleteCQ = scan.getAttribute(BaseScannerRegionObserver.DELETE_CQ);
      }
      emptyCF = scan.getAttribute(BaseScannerRegionObserver.EMPTY_CF);
    }

    int batchSize = 0;
    long ts = scan.getTimeRange().getMax();
    HRegion region = c.getEnvironment().getRegion();
    List<Mutation> mutations = Collections.emptyList();
    if (isDelete || isUpsert || (deleteCQ != null && deleteCF != null) || emptyCF != null) {
      // TODO: size better
      mutations = Lists.newArrayListWithExpectedSize(1024);
      batchSize =
          c.getEnvironment()
              .getConfiguration()
              .getInt(MUTATE_BATCH_SIZE_ATTRIB, QueryServicesOptions.DEFAULT_MUTATE_BATCH_SIZE);
    }
    Aggregators aggregators =
        ServerAggregators.deserialize(
            scan.getAttribute(BaseScannerRegionObserver.AGGREGATORS),
            c.getEnvironment().getConfiguration());
    Aggregator[] rowAggregators = aggregators.getAggregators();
    boolean hasMore;
    boolean hasAny = false;
    MultiKeyValueTuple result = new MultiKeyValueTuple();
    if (logger.isInfoEnabled()) {
      logger.info("Starting ungrouped coprocessor scan " + scan);
    }
    long rowCount = 0;
    region.startRegionOperation();
    try {
      do {
        List<Cell> results = new ArrayList<Cell>();
        // Results are potentially returned even when the return value of s.next is false
        // since this is an indication of whether or not there are more values after the
        // ones returned
        hasMore = innerScanner.nextRaw(results);
        if (!results.isEmpty()) {
          rowCount++;
          result.setKeyValues(results);
          try {
            if (isDelete) {
              // FIXME: the version of the Delete constructor without the lock args was introduced
              // in 0.94.4, thus if we try to use it here we can no longer use the 0.94.2 version
              // of the client.
              Cell firstKV = results.get(0);
              Delete delete =
                  new Delete(
                      firstKV.getRowArray(), firstKV.getRowOffset(), firstKV.getRowLength(), ts);
              mutations.add(delete);
            } else if (isUpsert) {
              Arrays.fill(values, null);
              int i = 0;
              List<PColumn> projectedColumns = projectedTable.getColumns();
              for (; i < projectedTable.getPKColumns().size(); i++) {
                Expression expression = selectExpressions.get(i);
                if (expression.evaluate(result, ptr)) {
                  values[i] = ptr.copyBytes();
                  // If SortOrder from expression in SELECT doesn't match the
                  // column being projected into then invert the bits.
                  if (expression.getSortOrder() != projectedColumns.get(i).getSortOrder()) {
                    SortOrder.invert(values[i], 0, values[i], 0, values[i].length);
                  }
                }
              }
              projectedTable.newKey(ptr, values);
              PRow row = projectedTable.newRow(kvBuilder, ts, ptr);
              for (; i < projectedColumns.size(); i++) {
                Expression expression = selectExpressions.get(i);
                if (expression.evaluate(result, ptr)) {
                  PColumn column = projectedColumns.get(i);
                  Object value = expression.getDataType().toObject(ptr, column.getSortOrder());
                  // We are guaranteed that the two column will have the same type.
                  if (!column
                      .getDataType()
                      .isSizeCompatible(
                          ptr,
                          value,
                          column.getDataType(),
                          expression.getMaxLength(),
                          expression.getScale(),
                          column.getMaxLength(),
                          column.getScale())) {
                    throw new ValueTypeIncompatibleException(
                        column.getDataType(), column.getMaxLength(), column.getScale());
                  }
                  column
                      .getDataType()
                      .coerceBytes(
                          ptr,
                          value,
                          expression.getDataType(),
                          expression.getMaxLength(),
                          expression.getScale(),
                          expression.getSortOrder(),
                          column.getMaxLength(),
                          column.getScale(),
                          column.getSortOrder());
                  byte[] bytes = ByteUtil.copyKeyBytesIfNecessary(ptr);
                  row.setValue(column, bytes);
                }
              }
              for (Mutation mutation : row.toRowMutations()) {
                mutations.add(mutation);
              }
            } else if (deleteCF != null && deleteCQ != null) {
              // No need to search for delete column, since we project only it
              // if no empty key value is being set
              if (emptyCF == null || result.getValue(deleteCF, deleteCQ) != null) {
                Delete delete =
                    new Delete(
                        results.get(0).getRowArray(),
                        results.get(0).getRowOffset(),
                        results.get(0).getRowLength());
                delete.deleteColumns(deleteCF, deleteCQ, ts);
                mutations.add(delete);
              }
            }
            if (emptyCF != null) {
              /*
               * If we've specified an emptyCF, then we need to insert an empty
               * key value "retroactively" for any key value that is visible at
               * the timestamp that the DDL was issued. Key values that are not
               * visible at this timestamp will not ever be projected up to
               * scans past this timestamp, so don't need to be considered.
               * We insert one empty key value per row per timestamp.
               */
              Set<Long> timeStamps = Sets.newHashSetWithExpectedSize(results.size());
              for (Cell kv : results) {
                long kvts = kv.getTimestamp();
                if (!timeStamps.contains(kvts)) {
                  Put put = new Put(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength());
                  put.add(
                      emptyCF, QueryConstants.EMPTY_COLUMN_BYTES, kvts, ByteUtil.EMPTY_BYTE_ARRAY);
                  mutations.add(put);
                }
              }
            }
            // Commit in batches based on UPSERT_BATCH_SIZE_ATTRIB in config
            if (!mutations.isEmpty() && batchSize > 0 && mutations.size() % batchSize == 0) {
              commitBatch(region, mutations, indexUUID);
              mutations.clear();
            }
          } catch (ConstraintViolationException e) {
            // Log and ignore in count
            logger.error(
                "Failed to create row in "
                    + region.getRegionNameAsString()
                    + " with values "
                    + SchemaUtil.toString(values),
                e);
            continue;
          }
          aggregators.aggregate(rowAggregators, result);
          hasAny = true;
        }
      } while (hasMore);
    } finally {
      innerScanner.close();
      region.closeRegionOperation();
    }

    if (logger.isInfoEnabled()) {
      logger.info("Finished scanning " + rowCount + " rows for ungrouped coprocessor scan " + scan);
    }

    if (!mutations.isEmpty()) {
      commitBatch(region, mutations, indexUUID);
    }

    final boolean hadAny = hasAny;
    KeyValue keyValue = null;
    if (hadAny) {
      byte[] value = aggregators.toBytes(rowAggregators);
      keyValue =
          KeyValueUtil.newKeyValue(
              UNGROUPED_AGG_ROW_KEY,
              SINGLE_COLUMN_FAMILY,
              SINGLE_COLUMN,
              AGG_TIMESTAMP,
              value,
              0,
              value.length);
    }
    final KeyValue aggKeyValue = keyValue;

    RegionScanner scanner =
        new BaseRegionScanner() {
          private boolean done = !hadAny;

          @Override
          public HRegionInfo getRegionInfo() {
            return innerScanner.getRegionInfo();
          }

          @Override
          public boolean isFilterDone() {
            return done;
          }

          @Override
          public void close() throws IOException {
            innerScanner.close();
          }

          @Override
          public boolean next(List<Cell> results) throws IOException {
            if (done) return false;
            done = true;
            results.add(aggKeyValue);
            return false;
          }

          @Override
          public long getMaxResultSize() {
            return scan.getMaxResultSize();
          }
        };
    return scanner;
  }