@Override
public void readFields(DataInput input) throws IOException {
int encodedByteLengthAndBool = WritableUtils.readVInt(input);
int byteLength = Math.abs(encodedByteLengthAndBool) - 1;
this.byteValue = new byte[byteLength];
input.readFully(byteValue, 0, byteLength);
int sortOrderAndDeterminism = WritableUtils.readVInt(input);
if (sortOrderAndDeterminism <= 2) {
// client is on an older version
this.determinism = encodedByteLengthAndBool > 0 ? Determinism.ALWAYS : Determinism.PER_ROW;
this.sortOrder = SortOrder.fromSystemValue(sortOrderAndDeterminism);
;
} else {
int determinismOrdinal = (sortOrderAndDeterminism >> 2) - 1;
this.determinism = Determinism.values()[determinismOrdinal];
int sortOrderValue =
sortOrderAndDeterminism & ((1 << 2) - 1); // get the least 2 significant bits
this.sortOrder = SortOrder.fromSystemValue(sortOrderValue);
}
int typeOrdinal = WritableUtils.readVInt(input);
if (typeOrdinal < 0) {
this.type = null;
} else {
this.type = PDataType.values()[typeOrdinal];
}
if (this.byteValue.length == 0) {
this.value = null;
} else {
this.value = this.type.toObject(byteValue, 0, byteValue.length, this.type, sortOrder);
}
}
/**
* Assigns the specified byte values to elements of the specified range of the specified array of
* bytes. The range to be filled extends from index fromIndex, inclusive, to index toIndex,
* exclusive. (If fromIndex==toIndex, the range to be filled is empty.)
*
* @param str the array to be filled
* @param strFromIdx the index of the first element (inclusive) to be filled with the fill values
* @param strToIdx the index of the last element (exclusive) to be filled with the fill values
* @param fillArray the values to be stored in all elements of the array
* @param fillFromIdx the index of the first element (inclusive) to be used as fill values
* @param filToIdx the index of the last element (exclusive) to be used as fill value
* @param invertFill if true inverts the bits in fill before filling the array
*/
public static void fill(
byte[] str,
int strFromIdx,
int strToIdx,
byte[] fillArray,
int fillFromIdx,
int fillToIdx,
boolean invertFill) {
rangeCheck(str.length, strFromIdx, strToIdx);
rangeCheck(fillArray.length, fillFromIdx, fillToIdx);
int strIdx = strFromIdx;
byte[] fill = fillArray;
int fillLen = fillToIdx - fillFromIdx;
if (invertFill) fill = SortOrder.invert(fillArray, fillFromIdx, fillLen);
while (strIdx < strToIdx) {
int fillIdx = fillFromIdx;
while (fillIdx < fillToIdx && strIdx < strToIdx) {
if (strIdx + fillLen < fillToIdx) {
System.arraycopy(fill, fillFromIdx, str, strIdx, fillLen);
} else {
str[strIdx++] = fill[fillIdx++];
}
}
}
}
private static RowKeySchema buildSchema(int[] widths) {
RowKeySchemaBuilder builder = new RowKeySchemaBuilder(10);
for (final int width : widths) {
builder.addField(
new PDatum() {
@Override
public boolean isNullable() {
return false;
}
@Override
public PDataType getDataType() {
return PChar.INSTANCE;
}
@Override
public Integer getMaxLength() {
return width;
}
@Override
public Integer getScale() {
return null;
}
@Override
public SortOrder getSortOrder() {
return SortOrder.getDefault();
}
},
false,
SortOrder.getDefault());
}
return builder.build();
}
private LiteralExpression(
Object value, PDataType type, byte[] byteValue, Determinism determinism) {
this(
value,
type,
byteValue,
type == null || !type.isFixedWidth() ? null : type.getMaxLength(value),
null,
SortOrder.getDefault(),
determinism);
}
private static int getBytesInCharNoException(byte b, SortOrder sortOrder) {
Preconditions.checkNotNull(sortOrder);
if (sortOrder == SortOrder.DESC) {
b = SortOrder.invert(b);
}
int c = b & 0xff;
if ((c & BYTES_1_MASK) == 0) return 1;
if ((c & BYTES_2_MASK) == 0xC0) return 2;
if ((c & BYTES_3_MASK) == 0xE0) return 3;
if ((c & BYTES_4_MASK) == 0xF0) return 4;
return -1;
}
@Test
public void testParseCreateTableInlinePrimaryKeyWithOrder() throws Exception {
for (String order : new String[] {"asc", "desc"}) {
String s =
"create table core.entity_history_archive (id char(15) primary key ${o})"
.replace("${o}", order);
CreateTableStatement stmt =
(CreateTableStatement) new SQLParser(new StringReader(s)).parseStatement();
List<ColumnDef> columnDefs = stmt.getColumnDefs();
assertEquals(1, columnDefs.size());
assertEquals(SortOrder.fromDDLValue(order), columnDefs.iterator().next().getSortOrder());
}
}
@Override
public void write(DataOutput output) throws IOException {
WritableUtils.writeVInt(
output, (byteValue.length + 1) * (this.determinism == Determinism.ALWAYS ? 1 : -1));
output.write(byteValue);
// since we need to support clients of a lower version, serialize the determinism enum ordinal
// in the int used to
// serialize sort order system value (which is either 1 or 2)
int sortOrderAndDeterminism =
((this.determinism.ordinal() + 1) << 2) + sortOrder.getSystemValue();
WritableUtils.writeVInt(output, sortOrderAndDeterminism);
WritableUtils.writeVInt(output, this.type == null ? -1 : this.type.ordinal());
}
@Override
public void readFields(DataInput input) throws IOException {
// read/write type ordinal, maxLength presence, scale presence and isNullable bit together to
// save space
int typeAndFlag = WritableUtils.readVInt(input);
isNullable = (typeAndFlag & 0x01) != 0;
if ((typeAndFlag & 0x02) != 0) {
scale = WritableUtils.readVInt(input);
}
if ((typeAndFlag & 0x04) != 0) {
maxLength = WritableUtils.readVInt(input);
}
type = PDataType.values()[typeAndFlag >>> 3];
sortOrder = SortOrder.fromSystemValue(WritableUtils.readVInt(input));
}
@Test
public void testParseCreateTablePrimaryKeyConstraintWithOrder() throws Exception {
for (String order : new String[] {"asc", "desc"}) {
String s =
"create table core.entity_history_archive (id CHAR(15), name VARCHAR(150), constraint pk primary key (id ${o}, name ${o}))"
.replace("${o}", order);
CreateTableStatement stmt =
(CreateTableStatement) new SQLParser(new StringReader(s)).parseStatement();
PrimaryKeyConstraint pkConstraint = stmt.getPrimaryKeyConstraint();
List<Pair<ColumnName, SortOrder>> columns = pkConstraint.getColumnNames();
assertEquals(2, columns.size());
for (Pair<ColumnName, SortOrder> pair : columns) {
assertEquals(
SortOrder.fromDDLValue(order), pkConstraint.getColumn(pair.getFirst()).getSecond());
}
}
}
@Override
public void write(DataOutput output) throws IOException {
// read/write type ordinal, maxLength presence, scale presence and isNullable bit together to
// save space
int typeAndFlag =
(isNullable ? 1 : 0)
| ((scale != null ? 1 : 0) << 1)
| ((maxLength != null ? 1 : 0) << 2)
| (type.ordinal() << 3);
WritableUtils.writeVInt(output, typeAndFlag);
if (scale != null) {
WritableUtils.writeVInt(output, scale);
}
if (maxLength != null) {
WritableUtils.writeVInt(output, maxLength);
}
WritableUtils.writeVInt(output, sortOrder.getSystemValue());
}
/**
* Static class for various schema-related utilities
*
* @since 0.1
*/
public class SchemaUtil {
private static final int VAR_LENGTH_ESTIMATE = 10;
private static final int VAR_KV_LENGTH_ESTIMATE = 50;
public static final String ESCAPE_CHARACTER = "\"";
public static final DataBlockEncoding DEFAULT_DATA_BLOCK_ENCODING = DataBlockEncoding.FAST_DIFF;
public static final PDatum VAR_BINARY_DATUM =
new PDatum() {
@Override
public boolean isNullable() {
return false;
}
@Override
public PDataType getDataType() {
return PVarbinary.INSTANCE;
}
@Override
public Integer getMaxLength() {
return null;
}
@Override
public Integer getScale() {
return null;
}
@Override
public SortOrder getSortOrder() {
return SortOrder.getDefault();
}
};
public static final RowKeySchema VAR_BINARY_SCHEMA =
new RowKeySchemaBuilder(1).addField(VAR_BINARY_DATUM, false, SortOrder.getDefault()).build();
/** May not be instantiated */
private SchemaUtil() {}
public static boolean isPKColumn(PColumn column) {
return column.getFamilyName() == null;
}
/**
* Imperfect estimate of row size given a PTable TODO: keep row count in stats table and use total
* size / row count instead
*
* @param table
* @return estimate of size in bytes of a row
*/
public static long estimateRowSize(PTable table) {
int keyLength = estimateKeyLength(table);
long rowSize = 0;
for (PColumn column : table.getColumns()) {
if (!SchemaUtil.isPKColumn(column)) {
PDataType type = column.getDataType();
Integer maxLength = column.getMaxLength();
int valueLength =
!type.isFixedWidth()
? VAR_KV_LENGTH_ESTIMATE
: maxLength == null ? type.getByteSize() : maxLength;
rowSize +=
KeyValue.getKeyValueDataStructureSize(
keyLength,
column.getFamilyName().getBytes().length,
column.getName().getBytes().length,
valueLength);
}
}
// Empty key value
rowSize +=
KeyValue.getKeyValueDataStructureSize(
keyLength,
getEmptyColumnFamily(table).length,
QueryConstants.EMPTY_COLUMN_BYTES.length,
0);
return rowSize;
}
/**
* Estimate the max key length in bytes of the PK for a given table
*
* @param table the table
* @return the max PK length
*/
public static int estimateKeyLength(PTable table) {
int maxKeyLength = 0;
// Calculate the max length of a key (each part must currently be of a fixed width)
int i = 0;
List<PColumn> columns = table.getPKColumns();
while (i < columns.size()) {
PColumn keyColumn = columns.get(i++);
PDataType type = keyColumn.getDataType();
Integer maxLength = keyColumn.getMaxLength();
maxKeyLength +=
!type.isFixedWidth()
? VAR_LENGTH_ESTIMATE
: maxLength == null ? type.getByteSize() : maxLength;
}
return maxKeyLength;
}
/**
* Normalize an identifier. If name is surrounded by double quotes, it is used as-is, otherwise
* the name is upper caased.
*
* @param name the parsed identifier
* @return the normalized identifier
*/
public static String normalizeIdentifier(String name) {
if (name == null) {
return name;
}
if (isCaseSensitive(name)) {
// Don't upper case if in quotes
return name.substring(1, name.length() - 1);
}
return name.toUpperCase();
}
/**
* Normalizes the fulltableName . Uses {@linkplain normalizeIdentifier}
*
* @param fullTableName
* @return
*/
public static String normalizeFullTableName(String fullTableName) {
String schemaName = SchemaUtil.getSchemaNameFromFullName(fullTableName);
String tableName = SchemaUtil.getTableNameFromFullName(fullTableName);
String normalizedTableName = StringUtil.EMPTY_STRING;
if (!schemaName.isEmpty()) {
normalizedTableName = normalizeIdentifier(schemaName) + QueryConstants.NAME_SEPARATOR;
}
return normalizedTableName + normalizeIdentifier(tableName);
}
public static boolean isCaseSensitive(String name) {
return name != null && name.length() > 0 && name.charAt(0) == '"';
}
public static <T> List<T> concat(List<T> l1, List<T> l2) {
int size1 = l1.size();
if (size1 == 0) {
return l2;
}
int size2 = l2.size();
if (size2 == 0) {
return l1;
}
List<T> l3 = new ArrayList<T>(size1 + size2);
l3.addAll(l1);
l3.addAll(l2);
return l3;
}
/**
* Get the key used in the Phoenix metadata row for a table definition
*
* @param schemaName
* @param tableName
*/
public static byte[] getTableKey(byte[] tenantId, byte[] schemaName, byte[] tableName) {
return ByteUtil.concat(
tenantId,
QueryConstants.SEPARATOR_BYTE_ARRAY,
schemaName,
QueryConstants.SEPARATOR_BYTE_ARRAY,
tableName);
}
/**
* Get the key used in the Phoenix function data row for a function definition
*
* @param tenantId
* @param functionName
*/
public static byte[] getFunctionKey(byte[] tenantId, byte[] functionName) {
return ByteUtil.concat(tenantId, QueryConstants.SEPARATOR_BYTE_ARRAY, functionName);
}
public static byte[] getTableKey(String tenantId, String schemaName, String tableName) {
return ByteUtil.concat(
tenantId == null ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(tenantId),
QueryConstants.SEPARATOR_BYTE_ARRAY,
schemaName == null ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(schemaName),
QueryConstants.SEPARATOR_BYTE_ARRAY,
Bytes.toBytes(tableName));
}
public static String getTableName(String schemaName, String tableName) {
return getName(schemaName, tableName, false);
}
private static String getName(String optionalQualifier, String name, boolean caseSensitive) {
String cq = caseSensitive ? "\"" + name + "\"" : name;
if (optionalQualifier == null || optionalQualifier.isEmpty()) {
return cq;
}
String cf = caseSensitive ? "\"" + optionalQualifier + "\"" : optionalQualifier;
return cf + QueryConstants.NAME_SEPARATOR + cq;
}
public static String getTableName(byte[] schemaName, byte[] tableName) {
return getName(schemaName, tableName);
}
public static String getColumnDisplayName(byte[] cf, byte[] cq) {
return getName(cf == null || cf.length == 0 ? ByteUtil.EMPTY_BYTE_ARRAY : cf, cq);
}
public static String getColumnDisplayName(String cf, String cq) {
return getName(cf == null || cf.isEmpty() ? null : cf, cq, false);
}
public static String getCaseSensitiveColumnDisplayName(String cf, String cq) {
return getName(cf == null || cf.isEmpty() ? null : cf, cq, true);
}
public static String getMetaDataEntityName(
String schemaName, String tableName, String familyName, String columnName) {
if ((schemaName == null || schemaName.isEmpty())
&& (tableName == null || tableName.isEmpty())) {
if (columnName == null || columnName.isEmpty()) {
return familyName;
}
return getName(familyName, columnName, false);
}
if ((familyName == null || familyName.isEmpty())
&& (columnName == null || columnName.isEmpty())) {
return getName(schemaName, tableName, false);
}
return getName(
getName(schemaName, tableName, false), getName(familyName, columnName, false), false);
}
public static String getColumnName(String familyName, String columnName) {
return getName(familyName, columnName, false);
}
public static byte[] getTableNameAsBytes(String schemaName, String tableName) {
if (schemaName == null || schemaName.length() == 0) {
return StringUtil.toBytes(tableName);
}
return getTableNameAsBytes(StringUtil.toBytes(schemaName), StringUtil.toBytes(tableName));
}
public static byte[] getTableNameAsBytes(byte[] schemaName, byte[] tableName) {
return getNameAsBytes(schemaName, tableName);
}
private static byte[] getNameAsBytes(byte[] nameOne, byte[] nameTwo) {
if (nameOne == null || nameOne.length == 0) {
return nameTwo;
} else if ((nameTwo == null || nameTwo.length == 0)) {
return nameOne;
} else {
return ByteUtil.concat(nameOne, QueryConstants.NAME_SEPARATOR_BYTES, nameTwo);
}
}
public static String getName(byte[] nameOne, byte[] nameTwo) {
return Bytes.toString(getNameAsBytes(nameOne, nameTwo));
}
public static int getVarCharLength(byte[] buf, int keyOffset, int maxLength) {
return getVarCharLength(buf, keyOffset, maxLength, 1);
}
public static int getVarCharLength(byte[] buf, int keyOffset, int maxLength, int skipCount) {
int length = 0;
for (int i = 0; i < skipCount; i++) {
while (length < maxLength && buf[keyOffset + length] != QueryConstants.SEPARATOR_BYTE) {
length++;
}
if (i != skipCount - 1) { // skip over the separator if it's not the last one.
length++;
}
}
return length;
}
public static int getVarChars(byte[] rowKey, byte[][] rowKeyMetadata) {
return getVarChars(rowKey, 0, rowKey.length, 0, rowKeyMetadata);
}
public static int getVarChars(byte[] rowKey, int colMetaDataLength, byte[][] colMetaData) {
return getVarChars(rowKey, 0, rowKey.length, 0, colMetaDataLength, colMetaData);
}
public static int getVarChars(
byte[] rowKey, int keyOffset, int keyLength, int colMetaDataOffset, byte[][] colMetaData) {
return getVarChars(
rowKey, keyOffset, keyLength, colMetaDataOffset, colMetaData.length, colMetaData);
}
public static int getVarChars(
byte[] rowKey,
int keyOffset,
int keyLength,
int colMetaDataOffset,
int colMetaDataLength,
byte[][] colMetaData) {
int i, offset = keyOffset;
for (i = colMetaDataOffset; i < colMetaDataLength && keyLength > 0; i++) {
int length = getVarCharLength(rowKey, offset, keyLength);
byte[] b = new byte[length];
System.arraycopy(rowKey, offset, b, 0, length);
offset += length + 1;
keyLength -= length + 1;
colMetaData[i] = b;
}
return i;
}
public static String findExistingColumn(PTable table, List<PColumn> columns) {
for (PColumn column : columns) {
PName familyName = column.getFamilyName();
if (familyName == null) {
try {
return table.getPKColumn(column.getName().getString()).getName().getString();
} catch (ColumnNotFoundException e) {
continue;
}
} else {
try {
return table
.getColumnFamily(familyName.getString())
.getColumn(column.getName().getString())
.getName()
.getString();
} catch (ColumnFamilyNotFoundException e) {
continue; // Shouldn't happen
} catch (ColumnNotFoundException e) {
continue;
}
}
}
return null;
}
public static String toString(byte[][] values) {
if (values == null) {
return "null";
}
StringBuilder buf = new StringBuilder("[");
for (byte[] value : values) {
buf.append(Bytes.toStringBinary(value));
buf.append(',');
}
buf.setCharAt(buf.length() - 1, ']');
return buf.toString();
}
public static String toString(PDataType type, byte[] value) {
boolean isString = type.isCoercibleTo(PVarchar.INSTANCE);
return isString
? ("'" + type.toObject(value).toString() + "'")
: type.toObject(value).toString();
}
public static byte[] getEmptyColumnFamily(
PName defaultColumnFamily, List<PColumnFamily> families) {
return families.isEmpty()
? defaultColumnFamily == null
? QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES
: defaultColumnFamily.getBytes()
: families.get(0).getName().getBytes();
}
public static byte[] getEmptyColumnFamily(PTable table) {
List<PColumnFamily> families = table.getColumnFamilies();
return families.isEmpty()
? table.getDefaultFamilyName() == null
? QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES
: table.getDefaultFamilyName().getBytes()
: families.get(0).getName().getBytes();
}
public static String getEmptyColumnFamilyAsString(PTable table) {
List<PColumnFamily> families = table.getColumnFamilies();
return families.isEmpty()
? table.getDefaultFamilyName() == null
? QueryConstants.DEFAULT_COLUMN_FAMILY
: table.getDefaultFamilyName().getString()
: families.get(0).getName().getString();
}
public static ImmutableBytesPtr getEmptyColumnFamilyPtr(PTable table) {
List<PColumnFamily> families = table.getColumnFamilies();
return families.isEmpty()
? table.getDefaultFamilyName() == null
? QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES_PTR
: table.getDefaultFamilyName().getBytesPtr()
: families.get(0).getName().getBytesPtr();
}
public static boolean isMetaTable(byte[] tableName) {
return Bytes.compareTo(tableName, SYSTEM_CATALOG_NAME_BYTES) == 0;
}
public static boolean isFunctionTable(byte[] tableName) {
return Bytes.compareTo(tableName, SYSTEM_FUNCTION_NAME_BYTES) == 0;
}
public static boolean isStatsTable(byte[] tableName) {
return Bytes.compareTo(tableName, SYSTEM_STATS_NAME_BYTES) == 0;
}
public static boolean isSequenceTable(byte[] tableName) {
return Bytes.compareTo(tableName, PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_NAME_BYTES) == 0;
}
public static boolean isSequenceTable(PTable table) {
return PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_NAME.equals(table.getName().getString());
}
public static boolean isMetaTable(PTable table) {
return PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA.equals(table.getSchemaName().getString())
&& PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE.equals(table.getTableName().getString());
}
public static boolean isMetaTable(byte[] schemaName, byte[] tableName) {
return Bytes.compareTo(schemaName, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA_BYTES) == 0
&& Bytes.compareTo(tableName, PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE_BYTES) == 0;
}
public static boolean isMetaTable(String schemaName, String tableName) {
return PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA.equals(schemaName)
&& PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE.equals(tableName);
}
public static boolean isSystemTable(byte[] fullTableName) {
String schemaName = SchemaUtil.getSchemaNameFromFullName(fullTableName);
if (QueryConstants.SYSTEM_SCHEMA_NAME.equals(schemaName)) return true;
return false;
}
// Given the splits and the rowKeySchema, find out the keys that
public static byte[][] processSplits(
byte[][] splits,
LinkedHashSet<PColumn> pkColumns,
Integer saltBucketNum,
boolean defaultRowKeyOrder)
throws SQLException {
// FIXME: shouldn't this return if splits.length == 0?
if (splits == null) return null;
// We do not accept user specified splits if the table is salted and we specify
// defaultRowKeyOrder. In this case,
// throw an exception.
if (splits.length > 0 && saltBucketNum != null && defaultRowKeyOrder) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_SPLITS_ON_SALTED_TABLE)
.build()
.buildException();
}
// If the splits are not specified and table is salted, pre-split the table.
if (splits.length == 0 && saltBucketNum != null) {
splits = SaltingUtil.getSalteByteSplitPoints(saltBucketNum);
}
byte[][] newSplits = new byte[splits.length][];
for (int i = 0; i < splits.length; i++) {
newSplits[i] = processSplit(splits[i], pkColumns);
}
return newSplits;
}
// Go through each slot in the schema and try match it with the split byte array. If the split
// does not confer to the schema, extends its length to match the schema.
private static byte[] processSplit(byte[] split, LinkedHashSet<PColumn> pkColumns) {
int pos = 0, offset = 0, maxOffset = split.length;
Iterator<PColumn> iterator = pkColumns.iterator();
while (pos < pkColumns.size()) {
PColumn column = iterator.next();
if (column.getDataType().isFixedWidth()) { // Fixed width
int length = SchemaUtil.getFixedByteSize(column);
if (maxOffset - offset < length) {
// The split truncates the field. Fill in the rest of the part and any fields that
// are missing after this field.
int fillInLength = length - (maxOffset - offset);
fillInLength += estimatePartLength(pos + 1, iterator);
return ByteUtil.fillKey(split, split.length + fillInLength);
}
// Account for this field, move to next position;
offset += length;
pos++;
} else { // Variable length
// If we are the last slot, then we are done. Nothing needs to be filled in.
if (pos == pkColumns.size() - 1) {
break;
}
while (offset < maxOffset && split[offset] != QueryConstants.SEPARATOR_BYTE) {
offset++;
}
if (offset == maxOffset) {
// The var-length field does not end with a separator and it's not the last field.
int fillInLength = 1; // SEPARATOR byte for the current var-length slot.
fillInLength += estimatePartLength(pos + 1, iterator);
return ByteUtil.fillKey(split, split.length + fillInLength);
}
// Move to the next position;
offset += 1; // skip separator;
pos++;
}
}
return split;
}
// Estimate the key length after pos slot for schema.
private static int estimatePartLength(int pos, Iterator<PColumn> iterator) {
int length = 0;
while (iterator.hasNext()) {
PColumn column = iterator.next();
if (column.getDataType().isFixedWidth()) {
length += SchemaUtil.getFixedByteSize(column);
} else {
length += 1; // SEPARATOR byte.
}
}
return length;
}
public static String getEscapedTableName(String schemaName, String tableName) {
if (schemaName == null || schemaName.length() == 0) {
return "\"" + tableName + "\"";
}
return "\"" + schemaName + "\"." + "\"" + tableName + "\"";
}
protected static PhoenixConnection addMetaDataColumn(
PhoenixConnection conn, long scn, String columnDef) throws SQLException {
PhoenixConnection metaConnection = null;
Statement stmt = null;
try {
metaConnection = new PhoenixConnection(conn.getQueryServices(), conn, scn);
try {
stmt = metaConnection.createStatement();
stmt.executeUpdate("ALTER TABLE SYSTEM.\"TABLE\" ADD IF NOT EXISTS " + columnDef);
return metaConnection;
} finally {
if (stmt != null) {
stmt.close();
}
}
} finally {
if (metaConnection != null) {
metaConnection.close();
}
}
}
public static boolean columnExists(PTable table, String columnName) {
try {
table.getColumn(columnName);
return true;
} catch (ColumnNotFoundException e) {
return false;
} catch (AmbiguousColumnException e) {
return true;
}
}
public static String getSchemaNameFromFullName(String tableName) {
int index = tableName.indexOf(QueryConstants.NAME_SEPARATOR);
if (index < 0) {
return StringUtil.EMPTY_STRING;
}
return tableName.substring(0, index);
}
private static int indexOf(byte[] bytes, byte b) {
for (int i = 0; i < bytes.length; i++) {
if (bytes[i] == b) {
return i;
}
}
return -1;
}
public static String getSchemaNameFromFullName(byte[] tableName) {
if (tableName == null) {
return null;
}
int index = indexOf(tableName, QueryConstants.NAME_SEPARATOR_BYTE);
if (index < 0) {
return StringUtil.EMPTY_STRING;
}
return Bytes.toString(tableName, 0, index);
}
public static String getTableNameFromFullName(byte[] tableName) {
if (tableName == null) {
return null;
}
int index = indexOf(tableName, QueryConstants.NAME_SEPARATOR_BYTE);
if (index < 0) {
return Bytes.toString(tableName);
}
return Bytes.toString(tableName, index + 1, tableName.length - index - 1);
}
public static String getTableNameFromFullName(String tableName) {
int index = tableName.indexOf(QueryConstants.NAME_SEPARATOR);
if (index < 0) {
return tableName;
}
return tableName.substring(index + 1, tableName.length());
}
public static byte[] getTableKeyFromFullName(String fullTableName) {
int index = fullTableName.indexOf(QueryConstants.NAME_SEPARATOR);
if (index < 0) {
return getTableKey(null, null, fullTableName);
}
String schemaName = fullTableName.substring(0, index);
String tableName = fullTableName.substring(index + 1);
return getTableKey(null, schemaName, tableName);
}
private static int getTerminatorCount(RowKeySchema schema) {
int nTerminators = 0;
for (int i = 0; i < schema.getFieldCount(); i++) {
Field field = schema.getField(i);
// We won't have a terminator on the last PK column
// unless it is variable length and exclusive, but
// having the extra byte irregardless won't hurt anything
if (!field.getDataType().isFixedWidth()) {
nTerminators++;
}
}
return nTerminators;
}
public static int getMaxKeyLength(RowKeySchema schema, List<List<KeyRange>> slots) {
int maxKeyLength = getTerminatorCount(schema) * 2;
for (List<KeyRange> slot : slots) {
int maxSlotLength = 0;
for (KeyRange range : slot) {
int maxRangeLength = Math.max(range.getLowerRange().length, range.getUpperRange().length);
if (maxSlotLength < maxRangeLength) {
maxSlotLength = maxRangeLength;
}
}
maxKeyLength += maxSlotLength;
}
return maxKeyLength;
}
public static int getFixedByteSize(PDatum e) {
assert (e.getDataType().isFixedWidth());
Integer maxLength = e.getMaxLength();
return maxLength == null ? e.getDataType().getByteSize() : maxLength;
}
public static short getMaxKeySeq(PTable table) {
int offset = 0;
if (table.getBucketNum() != null) {
offset++;
}
// TODO: for tenant-specific table on tenant-specific connection,
// we should subtract one for tenant column and another one for
// index ID
return (short) (table.getPKColumns().size() - offset);
}
public static int getPKPosition(PTable table, PColumn column) {
// TODO: when PColumn has getPKPosition, use that instead
return table.getPKColumns().indexOf(column);
}
public static String getEscapedFullColumnName(String fullColumnName) {
if (fullColumnName.startsWith(ESCAPE_CHARACTER)) {
return fullColumnName;
}
int index = fullColumnName.indexOf(QueryConstants.NAME_SEPARATOR);
if (index < 0) {
return getEscapedArgument(fullColumnName);
}
String columnFamily = fullColumnName.substring(0, index);
String columnName = fullColumnName.substring(index + 1);
return getEscapedArgument(columnFamily)
+ QueryConstants.NAME_SEPARATOR
+ getEscapedArgument(columnName);
}
public static String getEscapedFullTableName(String fullTableName) {
final String schemaName = getSchemaNameFromFullName(fullTableName);
final String tableName = getTableNameFromFullName(fullTableName);
return getEscapedTableName(schemaName, tableName);
}
/**
* Escapes the given argument with {@value #ESCAPE_CHARACTER}
*
* @param argument any non null value.
* @return
*/
public static String getEscapedArgument(String argument) {
Preconditions.checkNotNull(argument, "Argument passed cannot be null");
return ESCAPE_CHARACTER + argument + ESCAPE_CHARACTER;
}
/**
* @return a fully qualified column name in the format: "CFNAME"."COLNAME" or "COLNAME" depending
* on whether or not there is a column family name present.
*/
public static String getQuotedFullColumnName(PColumn pCol) {
checkNotNull(pCol);
String columnName = pCol.getName().getString();
String columnFamilyName =
pCol.getFamilyName() != null ? pCol.getFamilyName().getString() : null;
return getQuotedFullColumnName(columnFamilyName, columnName);
}
/**
* @return a fully qualified column name in the format: "CFNAME"."COLNAME" or "COLNAME" depending
* on whether or not there is a column family name present.
*/
public static String getQuotedFullColumnName(
@Nullable String columnFamilyName, String columnName) {
checkArgument(!isNullOrEmpty(columnName), "Column name cannot be null or empty");
return columnFamilyName == null
? ("\"" + columnName + "\"")
: ("\""
+ columnFamilyName
+ "\""
+ QueryConstants.NAME_SEPARATOR
+ "\""
+ columnName
+ "\"");
}
public static boolean hasHTableDescriptorProps(Map<String, Object> tableProps) {
int pTablePropCount = 0;
for (String prop : tableProps.keySet()) {
if (TableProperty.isPhoenixTableProperty(prop)) {
pTablePropCount++;
}
}
return tableProps.size() - pTablePropCount > 0;
}
/**
* Replaces all occurrences of {@link #ESCAPE_CHARACTER} with an empty character.
*
* @param fullColumnName
* @return
*/
public static String getUnEscapedFullColumnName(String fullColumnName) {
checkArgument(!isNullOrEmpty(fullColumnName), "Column name cannot be null or empty");
fullColumnName = fullColumnName.replaceAll(ESCAPE_CHARACTER, "");
return fullColumnName.trim();
}
/**
* Return the separator byte to use based on:
*
* @param rowKeyOrderOptimizable whether or not the table may optimize descending row keys. If the
* table has no descending row keys, this will be true. Also, if the table has been upgraded
* (using a new -u option for psql.py), then it'll be true
* @param isNullValue whether or not the value is null. We use a null byte still if the value is
* null regardless of sort order since nulls will always sort first this way.
* @param sortOrder whether the value sorts ascending or descending.
* @return the byte to use as the separator
*/
public static byte getSeparatorByte(
boolean rowKeyOrderOptimizable, boolean isNullValue, SortOrder sortOrder) {
return !rowKeyOrderOptimizable || isNullValue || sortOrder == SortOrder.ASC
? QueryConstants.SEPARATOR_BYTE
: QueryConstants.DESC_SEPARATOR_BYTE;
}
public static byte getSeparatorByte(
boolean rowKeyOrderOptimizable, boolean isNullValue, Field f) {
return getSeparatorByte(rowKeyOrderOptimizable, isNullValue, f.getSortOrder());
}
public static byte getSeparatorByte(
boolean rowKeyOrderOptimizable, boolean isNullValue, Expression e) {
return getSeparatorByte(rowKeyOrderOptimizable, isNullValue, e.getSortOrder());
}
/**
* Get list of ColumnInfos that contain Column Name and its associated PDataType for an import.
* The supplied list of columns can be null -- if it is non-null, it represents a user-supplied
* list of columns to be imported.
*
* @param conn Phoenix connection from which metadata will be read
* @param tableName Phoenix table name whose columns are to be checked. Can include a schema name
* @param columns user-supplied list of import columns, can be null
* @param strict if true, an exception will be thrown if unknown columns are supplied
*/
public static List<ColumnInfo> generateColumnInfo(
Connection conn, String tableName, List<String> columns, boolean strict) throws SQLException {
Map<String, Integer> columnNameToTypeMap = Maps.newLinkedHashMap();
Set<String> ambiguousColumnNames = new HashSet<String>();
Map<String, Integer> fullColumnNameToTypeMap = Maps.newLinkedHashMap();
DatabaseMetaData dbmd = conn.getMetaData();
int unfoundColumnCount = 0;
// TODO: escape wildcard characters here because we don't want that
// behavior here
String escapedTableName = StringUtil.escapeLike(tableName);
String[] schemaAndTable = escapedTableName.split("\\.");
ResultSet rs = null;
try {
rs =
dbmd.getColumns(
null,
(schemaAndTable.length == 1 ? "" : schemaAndTable[0]),
(schemaAndTable.length == 1 ? escapedTableName : schemaAndTable[1]),
null);
while (rs.next()) {
String colName = rs.getString(QueryUtil.COLUMN_NAME_POSITION);
String colFam = rs.getString(QueryUtil.COLUMN_FAMILY_POSITION);
// use family qualifier, if available, otherwise, use column name
String fullColumn = (colFam == null ? colName : String.format("%s.%s", colFam, colName));
String sqlTypeName = rs.getString(QueryUtil.DATA_TYPE_NAME_POSITION);
// allow for both bare and family qualified names.
if (columnNameToTypeMap.keySet().contains(colName)) {
ambiguousColumnNames.add(colName);
}
columnNameToTypeMap.put(colName, PDataType.fromSqlTypeName(sqlTypeName).getSqlType());
fullColumnNameToTypeMap.put(
fullColumn, PDataType.fromSqlTypeName(sqlTypeName).getSqlType());
}
if (columnNameToTypeMap.isEmpty()) {
throw new IllegalArgumentException("Table " + tableName + " not found");
}
} finally {
if (rs != null) {
rs.close();
}
}
List<ColumnInfo> columnInfoList = Lists.newArrayList();
Set<String> unresolvedColumnNames = new TreeSet<String>();
if (columns == null) {
// use family qualified names by default, if no columns are specified.
for (Map.Entry<String, Integer> entry : fullColumnNameToTypeMap.entrySet()) {
columnInfoList.add(new ColumnInfo(entry.getKey(), entry.getValue()));
}
} else {
// Leave "null" as indication to skip b/c it doesn't exist
for (int i = 0; i < columns.size(); i++) {
String columnName = columns.get(i).trim();
Integer sqlType = null;
if (fullColumnNameToTypeMap.containsKey(columnName)) {
sqlType = fullColumnNameToTypeMap.get(columnName);
} else if (columnNameToTypeMap.containsKey(columnName)) {
if (ambiguousColumnNames.contains(columnName)) {
unresolvedColumnNames.add(columnName);
}
// fall back to bare column name.
sqlType = columnNameToTypeMap.get(columnName);
}
if (unresolvedColumnNames.size() > 0) {
StringBuilder exceptionMessage = new StringBuilder();
boolean first = true;
exceptionMessage.append(
"Unable to resolve these column names to a single column family:\n");
for (String col : unresolvedColumnNames) {
if (first) first = false;
else exceptionMessage.append(",");
exceptionMessage.append(col);
}
exceptionMessage.append("\nAvailable columns with column families:\n");
first = true;
for (String col : fullColumnNameToTypeMap.keySet()) {
if (first) first = false;
else exceptionMessage.append(",");
exceptionMessage.append(col);
}
throw new SQLException(exceptionMessage.toString());
}
if (sqlType == null) {
if (strict) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.COLUMN_NOT_FOUND)
.setColumnName(columnName)
.setTableName(tableName)
.build()
.buildException();
}
unfoundColumnCount++;
} else {
columnInfoList.add(new ColumnInfo(columnName, sqlType));
}
}
if (unfoundColumnCount == columns.size()) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.COLUMN_NOT_FOUND)
.setColumnName(Arrays.toString(columns.toArray(new String[0])))
.setTableName(tableName)
.build()
.buildException();
}
}
return columnInfoList;
}
public static boolean hasRowTimestampColumn(PTable table) {
return table.getRowTimestampColPos() > 0;
}
public static byte[] getTableKey(PTable dataTable) {
PName tenantId = dataTable.getTenantId();
PName schemaName = dataTable.getSchemaName();
return getTableKey(
tenantId == null ? ByteUtil.EMPTY_BYTE_ARRAY : tenantId.getBytes(),
schemaName == null ? ByteUtil.EMPTY_BYTE_ARRAY : schemaName.getBytes(),
dataTable.getTableName().getBytes());
}
}
@Override
public SortOrder getSortOrder() {
return SortOrder.getDefault();
}
public class StringUtil {
public static final String EMPTY_STRING = "";
// Masks to determine how many bytes are in each character
// From http://tools.ietf.org/html/rfc3629#section-3
public static final byte SPACE_UTF8 = 0x20;
private static final int BYTES_1_MASK = 0xFF << 7; // 0xxxxxxx is a single byte char
private static final int BYTES_2_MASK = 0xFF << 5; // 110xxxxx is a double byte char
private static final int BYTES_3_MASK = 0xFF << 4; // 1110xxxx is a triple byte char
private static final int BYTES_4_MASK = 0xFF << 3; // 11110xxx is a quadruple byte char
public static final byte INVERTED_SPACE_UTF8 =
SortOrder.invert(new byte[] {SPACE_UTF8}, 0, new byte[1], 0, 1)[0];
public static final char SINGLE_CHAR_WILDCARD = '?';
public static final char SINGLE_CHAR_LIKE = '_';
public static final char MULTI_CHAR_WILDCARD = '*';
public static final char MULTI_CHAR_LIKE = '%';
public static final String[] LIKE_ESCAPE_SEQS =
new String[] {"\\" + SINGLE_CHAR_LIKE, "\\" + MULTI_CHAR_LIKE};
public static final String[] LIKE_UNESCAPED_SEQS =
new String[] {"" + SINGLE_CHAR_LIKE, "" + MULTI_CHAR_LIKE};
private StringUtil() {}
/** Replace instances of character ch in String value with String replacement */
public static String replaceChar(String value, char ch, CharSequence replacement) {
if (value == null) return null;
int i = value.indexOf(ch);
if (i == -1) return value; // nothing to do
// we've got at least one character to replace
StringBuilder buf = new StringBuilder(value.length() + 16); // some extra space
int j = 0;
while (i != -1) {
buf.append(value, j, i).append(replacement);
j = i + 1;
i = value.indexOf(ch, j);
}
if (j < value.length()) buf.append(value, j, value.length());
return buf.toString();
}
/**
* @return the replacement of all occurrences of src[i] with target[i] in s. Src and target are
* not regex's so this uses simple searching with indexOf()
*/
public static String replace(String s, String[] src, String[] target) {
assert src != null && target != null && src.length > 0 && src.length == target.length;
if (src.length == 1 && src[0].length() == 1) {
return replaceChar(s, src[0].charAt(0), target[0]);
}
if (s == null) return null;
StringBuilder sb = new StringBuilder(s.length());
int pos = 0;
int limit = s.length();
int lastMatch = 0;
while (pos < limit) {
boolean matched = false;
for (int i = 0; i < src.length; i++) {
if (s.startsWith(src[i], pos) && src[i].length() > 0) {
// we found a matching pattern - append the acculumation plus the replacement
sb.append(s.substring(lastMatch, pos)).append(target[i]);
pos += src[i].length();
lastMatch = pos;
matched = true;
break;
}
}
if (!matched) {
// we didn't match any patterns, so move forward 1 character
pos++;
}
}
// see if we found any matches
if (lastMatch == 0) {
// we didn't match anything, so return the source string
return s;
}
// apppend the trailing portion
sb.append(s.substring(lastMatch));
return sb.toString();
}
public static int getBytesInChar(byte b, SortOrder sortOrder) {
int ret = getBytesInCharNoException(b, sortOrder);
if (ret == -1) throw new UndecodableByteException(b);
return ret;
}
private static int getBytesInCharNoException(byte b, SortOrder sortOrder) {
Preconditions.checkNotNull(sortOrder);
if (sortOrder == SortOrder.DESC) {
b = SortOrder.invert(b);
}
int c = b & 0xff;
if ((c & BYTES_1_MASK) == 0) return 1;
if ((c & BYTES_2_MASK) == 0xC0) return 2;
if ((c & BYTES_3_MASK) == 0xE0) return 3;
if ((c & BYTES_4_MASK) == 0xF0) return 4;
return -1;
}
public static int calculateUTF8Length(byte[] bytes, int offset, int length, SortOrder sortOrder) {
int i = offset, endOffset = offset + length;
length = 0;
while (i < endOffset) {
int charLength = getBytesInChar(bytes[i], sortOrder);
i += charLength;
length++;
}
return length;
}
// given an array of bytes containing utf-8 encoded strings, starting from curPos, ending before
// range, and return the next character offset, -1 if no next character available or
// UndecodableByteException
private static int calculateNextCharOffset(
byte[] bytes, int curPos, int range, SortOrder sortOrder) {
int ret = getBytesInCharNoException(bytes[curPos], sortOrder);
if (ret == -1) return -1;
ret += curPos;
if (ret >= range) return -1;
return ret;
}
// given an array of bytes containing utf-8 encoded strings, starting from offset, and return
// the previous character offset , -1 if UndecodableByteException. curPos points to current
// character starting offset.
private static int calculatePreCharOffset(
byte[] bytes, int curPos, int offset, SortOrder sortOrder) {
--curPos;
for (int i = 1, pos = curPos - i + 1; i <= 4 && offset <= pos; ++i, --pos) {
int ret = getBytesInCharNoException(bytes[pos], sortOrder);
if (ret == i) return pos;
}
return -1;
}
// return actural offsetInBytes corresponding to offsetInStr in utf-8 encoded strings bytes
// containing
// @param bytes an array of bytes containing utf-8 encoded strings
// @param offset
// @param length
// @param sortOrder
// @param offsetInStr offset for utf-8 encoded strings bytes array containing. Can be negative
// meaning counting from the end of encoded strings
// @return actural offsetInBytes corresponding to offsetInStr. -1 if offsetInStr is out of index
public static int calculateUTF8Offset(
byte[] bytes, int offset, int length, SortOrder sortOrder, int offsetInStr) {
if (offsetInStr == 0) return offset;
int ret, range = offset + length;
if (offsetInStr > 0) {
ret = offset;
while (offsetInStr > 0) {
ret = calculateNextCharOffset(bytes, ret, range, sortOrder);
if (ret == -1) return -1;
--offsetInStr;
}
} else {
ret = offset + length;
while (offsetInStr < 0) {
ret = calculatePreCharOffset(bytes, ret, offset, sortOrder);
// if calculateCurCharOffset returns -1, ret must be smaller than offset
if (ret < offset) return -1;
++offsetInStr;
}
}
return ret;
}
// Given an array of bytes containing encoding utf-8 encoded strings, the offset and a length
// parameter, return the actual index into the byte array which would represent a substring
// of <length> starting from the character at <offset>. We assume the <offset> is the start
// byte of an UTF-8 character.
public static int getByteLengthForUtf8SubStr(
byte[] bytes, int offset, int length, SortOrder sortOrder) {
int byteLength = 0;
while (length > 0 && offset + byteLength < bytes.length) {
int charLength = getBytesInChar(bytes[offset + byteLength], sortOrder);
byteLength += charLength;
length--;
}
return byteLength;
}
public static boolean hasMultiByteChars(String s) {
for (int i = 0; i < s.length(); i++) {
char c = s.charAt(i);
if (c > 0x007F) {
return true;
}
}
return false;
}
public static int getFirstNonBlankCharIdxFromStart(
byte[] string, int offset, int length, SortOrder sortOrder) {
int i = offset;
byte space = sortOrder == SortOrder.ASC ? SPACE_UTF8 : INVERTED_SPACE_UTF8;
for (; i < offset + length; i++) {
if (string[i] != space) {
break;
}
}
return i;
}
public static int getFirstNonBlankCharIdxFromEnd(
byte[] string, int offset, int length, SortOrder sortOrder) {
int i = offset + length - 1;
byte space = sortOrder == SortOrder.ASC ? SPACE_UTF8 : INVERTED_SPACE_UTF8;
for (; i >= offset; i--) {
if (string[i] != space) {
break;
}
}
return i;
}
// A toBytes function backed up HBase's utility function, but would accept null input, in which
// case it returns an empty byte array.
public static byte[] toBytes(String input) {
if (input == null) {
return ByteUtil.EMPTY_BYTE_ARRAY;
}
return Bytes.toBytes(input);
}
public static String escapeLike(String s) {
return replace(s, LIKE_UNESCAPED_SEQS, LIKE_ESCAPE_SEQS);
}
public static int getUnpaddedCharLength(byte[] b, int offset, int length, SortOrder sortOrder) {
return getFirstNonBlankCharIdxFromEnd(b, offset, length, sortOrder) - offset + 1;
}
public static byte[] padChar(byte[] value, int offset, int length, int paddedLength) {
byte[] key = new byte[paddedLength];
System.arraycopy(value, offset, key, 0, length);
Arrays.fill(key, length, paddedLength, SPACE_UTF8);
return key;
}
public static byte[] padChar(byte[] value, Integer byteSize) {
byte[] newValue = Arrays.copyOf(value, byteSize);
if (newValue.length > value.length) {
Arrays.fill(newValue, value.length, newValue.length, SPACE_UTF8);
}
return newValue;
}
/**
* Lame - StringBuilder.equals is retarded.
*
* @param b1
* @param b2
* @return whether or not the two builders consist the same sequence of characters
*/
public static boolean equals(StringBuilder b1, StringBuilder b2) {
if (b1.length() != b2.length()) {
return false;
}
for (int i = 0; i < b1.length(); i++) {
if (b1.charAt(i) != b2.charAt(i)) {
return false;
}
}
return true;
}
/**
* LPAD implementation
*
* @param str array containing string to be left padded
* @param strOffset byte offset of string
* @param strLength byte length of string
* @param fill array containing fill values
* @param fillOffset byte offset of fill
* @param fillLength byte length of fill
* @param invertFill if true inverts the bits in fill before filling the array
* @param strWithPaddingLen length of the string that is returned with fill values left padded
* @return byte[] containing left padded string
*/
public static byte[] lpad(
byte[] str,
int strOffset,
int strLength,
byte[] fill,
int fillOffset,
int fillLength,
boolean invertFill,
int strWithPaddingLen) {
byte[] paddedStr = new byte[strWithPaddingLen];
int fillStopIdx = strWithPaddingLen - strLength;
// copy fill into the start of paddedStr
fill(paddedStr, 0, fillStopIdx, fill, fillOffset, fillOffset + fillLength, invertFill);
// fill remaining characters with original string
System.arraycopy(str, strOffset, paddedStr, fillStopIdx, strLength);
return paddedStr;
}
/**
* Assigns the specified byte values to elements of the specified range of the specified array of
* bytes. The range to be filled extends from index fromIndex, inclusive, to index toIndex,
* exclusive. (If fromIndex==toIndex, the range to be filled is empty.)
*
* @param str the array to be filled
* @param strFromIdx the index of the first element (inclusive) to be filled with the fill values
* @param strToIdx the index of the last element (exclusive) to be filled with the fill values
* @param fillArray the values to be stored in all elements of the array
* @param fillFromIdx the index of the first element (inclusive) to be used as fill values
* @param filToIdx the index of the last element (exclusive) to be used as fill value
* @param invertFill if true inverts the bits in fill before filling the array
*/
public static void fill(
byte[] str,
int strFromIdx,
int strToIdx,
byte[] fillArray,
int fillFromIdx,
int fillToIdx,
boolean invertFill) {
rangeCheck(str.length, strFromIdx, strToIdx);
rangeCheck(fillArray.length, fillFromIdx, fillToIdx);
int strIdx = strFromIdx;
byte[] fill = fillArray;
int fillLen = fillToIdx - fillFromIdx;
if (invertFill) fill = SortOrder.invert(fillArray, fillFromIdx, fillLen);
while (strIdx < strToIdx) {
int fillIdx = fillFromIdx;
while (fillIdx < fillToIdx && strIdx < strToIdx) {
if (strIdx + fillLen < fillToIdx) {
System.arraycopy(fill, fillFromIdx, str, strIdx, fillLen);
} else {
str[strIdx++] = fill[fillIdx++];
}
}
}
}
/**
* Checks that fromIndex and toIndex are in the range and throws an appropriate exception, if they
* are not
*/
private static void rangeCheck(int length, int fromIndex, int toIndex) {
if (fromIndex > toIndex) {
throw new IllegalArgumentException("fromIndex(" + fromIndex + ") > toIndex(" + toIndex + ")");
}
if (fromIndex < 0) {
throw new ArrayIndexOutOfBoundsException(fromIndex);
}
if (toIndex > length) {
throw new ArrayIndexOutOfBoundsException(toIndex);
}
}
public static String escapeStringConstant(String pattern) {
return StringEscapeUtils.escapeSql(pattern); // Need to escape double quotes
}
public static String escapeBackslash(String input) {
// see
// http://stackoverflow.com/questions/4653831/regex-how-to-escape-backslashes-and-special-characters
return input.replaceAll("\\\\", "\\\\\\\\");
}
}
public static LiteralExpression newConstant(
Object value, PDataType type, Integer maxLength, Integer scale, Determinism determinism)
throws SQLException { // remove?
return newConstant(value, type, maxLength, scale, SortOrder.getDefault(), determinism);
}
public static LiteralExpression newConstant(Object value, PDataType type, Determinism determinism)
throws SQLException {
return newConstant(value, type, SortOrder.getDefault(), determinism);
}
private static MutationState upsertSelect(
PhoenixStatement statement,
TableRef tableRef,
RowProjector projector,
ResultIterator iterator,
int[] columnIndexes,
int[] pkSlotIndexes)
throws SQLException {
try {
PhoenixConnection connection = statement.getConnection();
ConnectionQueryServices services = connection.getQueryServices();
int maxSize =
services
.getProps()
.getInt(
QueryServices.MAX_MUTATION_SIZE_ATTRIB,
QueryServicesOptions.DEFAULT_MAX_MUTATION_SIZE);
int batchSize = Math.min(connection.getMutateBatchSize(), maxSize);
boolean isAutoCommit = connection.getAutoCommit();
byte[][] values = new byte[columnIndexes.length][];
int rowCount = 0;
Map<ImmutableBytesPtr, Map<PColumn, byte[]>> mutation =
Maps.newHashMapWithExpectedSize(batchSize);
PTable table = tableRef.getTable();
ResultSet rs = new PhoenixResultSet(iterator, projector, statement);
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
while (rs.next()) {
for (int i = 0; i < values.length; i++) {
PColumn column = table.getColumns().get(columnIndexes[i]);
byte[] bytes = rs.getBytes(i + 1);
ptr.set(bytes == null ? ByteUtil.EMPTY_BYTE_ARRAY : bytes);
Object value = rs.getObject(i + 1);
int rsPrecision = rs.getMetaData().getPrecision(i + 1);
Integer precision = rsPrecision == 0 ? null : rsPrecision;
int rsScale = rs.getMetaData().getScale(i + 1);
Integer scale = rsScale == 0 ? null : rsScale;
// We are guaranteed that the two column will have compatible types,
// as we checked that before.
if (!column
.getDataType()
.isSizeCompatible(
ptr,
value,
column.getDataType(),
precision,
scale,
column.getMaxLength(),
column.getScale())) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.DATA_EXCEEDS_MAX_CAPACITY)
.setColumnName(column.getName().getString())
.setMessage("value=" + column.getDataType().toStringLiteral(ptr, null))
.build()
.buildException();
}
column
.getDataType()
.coerceBytes(
ptr,
value,
column.getDataType(),
precision,
scale,
SortOrder.getDefault(),
column.getMaxLength(),
column.getScale(),
column.getSortOrder());
values[i] = ByteUtil.copyKeyBytesIfNecessary(ptr);
}
setValues(values, pkSlotIndexes, columnIndexes, table, mutation);
rowCount++;
// Commit a batch if auto commit is true and we're at our batch size
if (isAutoCommit && rowCount % batchSize == 0) {
MutationState state = new MutationState(tableRef, mutation, 0, maxSize, connection);
connection.getMutationState().join(state);
connection.commit();
mutation.clear();
}
}
// If auto commit is true, this last batch will be committed upon return
return new MutationState(
tableRef, mutation, rowCount / batchSize * batchSize, maxSize, connection);
} finally {
iterator.close();
}
}
@Override
protected RegionScanner doPostScannerOpen(
final ObserverContext<RegionCoprocessorEnvironment> c, final Scan scan, final RegionScanner s)
throws IOException {
byte[] isUngroupedAgg = scan.getAttribute(BaseScannerRegionObserver.UNGROUPED_AGG);
if (isUngroupedAgg == null) {
return s;
}
final ScanProjector p = ScanProjector.deserializeProjectorFromScan(scan);
final HashJoinInfo j = HashJoinInfo.deserializeHashJoinFromScan(scan);
RegionScanner theScanner = s;
if (p != null || j != null) {
theScanner =
new HashJoinRegionScanner(s, p, j, ScanUtil.getTenantId(scan), c.getEnvironment());
}
final RegionScanner innerScanner = theScanner;
byte[] indexUUID = scan.getAttribute(PhoenixIndexCodec.INDEX_UUID);
PTable projectedTable = null;
List<Expression> selectExpressions = null;
byte[] upsertSelectTable = scan.getAttribute(BaseScannerRegionObserver.UPSERT_SELECT_TABLE);
boolean isUpsert = false;
boolean isDelete = false;
byte[] deleteCQ = null;
byte[] deleteCF = null;
byte[][] values = null;
byte[] emptyCF = null;
ImmutableBytesWritable ptr = null;
if (upsertSelectTable != null) {
isUpsert = true;
projectedTable = deserializeTable(upsertSelectTable);
selectExpressions =
deserializeExpressions(scan.getAttribute(BaseScannerRegionObserver.UPSERT_SELECT_EXPRS));
values = new byte[projectedTable.getPKColumns().size()][];
ptr = new ImmutableBytesWritable();
} else {
byte[] isDeleteAgg = scan.getAttribute(BaseScannerRegionObserver.DELETE_AGG);
isDelete = isDeleteAgg != null && Bytes.compareTo(PDataType.TRUE_BYTES, isDeleteAgg) == 0;
if (!isDelete) {
deleteCF = scan.getAttribute(BaseScannerRegionObserver.DELETE_CF);
deleteCQ = scan.getAttribute(BaseScannerRegionObserver.DELETE_CQ);
}
emptyCF = scan.getAttribute(BaseScannerRegionObserver.EMPTY_CF);
}
int batchSize = 0;
long ts = scan.getTimeRange().getMax();
HRegion region = c.getEnvironment().getRegion();
List<Mutation> mutations = Collections.emptyList();
if (isDelete || isUpsert || (deleteCQ != null && deleteCF != null) || emptyCF != null) {
// TODO: size better
mutations = Lists.newArrayListWithExpectedSize(1024);
batchSize =
c.getEnvironment()
.getConfiguration()
.getInt(MUTATE_BATCH_SIZE_ATTRIB, QueryServicesOptions.DEFAULT_MUTATE_BATCH_SIZE);
}
Aggregators aggregators =
ServerAggregators.deserialize(
scan.getAttribute(BaseScannerRegionObserver.AGGREGATORS),
c.getEnvironment().getConfiguration());
Aggregator[] rowAggregators = aggregators.getAggregators();
boolean hasMore;
boolean hasAny = false;
MultiKeyValueTuple result = new MultiKeyValueTuple();
if (logger.isInfoEnabled()) {
logger.info("Starting ungrouped coprocessor scan " + scan);
}
long rowCount = 0;
region.startRegionOperation();
try {
do {
List<Cell> results = new ArrayList<Cell>();
// Results are potentially returned even when the return value of s.next is false
// since this is an indication of whether or not there are more values after the
// ones returned
hasMore = innerScanner.nextRaw(results);
if (!results.isEmpty()) {
rowCount++;
result.setKeyValues(results);
try {
if (isDelete) {
// FIXME: the version of the Delete constructor without the lock args was introduced
// in 0.94.4, thus if we try to use it here we can no longer use the 0.94.2 version
// of the client.
Cell firstKV = results.get(0);
Delete delete =
new Delete(
firstKV.getRowArray(), firstKV.getRowOffset(), firstKV.getRowLength(), ts);
mutations.add(delete);
} else if (isUpsert) {
Arrays.fill(values, null);
int i = 0;
List<PColumn> projectedColumns = projectedTable.getColumns();
for (; i < projectedTable.getPKColumns().size(); i++) {
Expression expression = selectExpressions.get(i);
if (expression.evaluate(result, ptr)) {
values[i] = ptr.copyBytes();
// If SortOrder from expression in SELECT doesn't match the
// column being projected into then invert the bits.
if (expression.getSortOrder() != projectedColumns.get(i).getSortOrder()) {
SortOrder.invert(values[i], 0, values[i], 0, values[i].length);
}
}
}
projectedTable.newKey(ptr, values);
PRow row = projectedTable.newRow(kvBuilder, ts, ptr);
for (; i < projectedColumns.size(); i++) {
Expression expression = selectExpressions.get(i);
if (expression.evaluate(result, ptr)) {
PColumn column = projectedColumns.get(i);
Object value = expression.getDataType().toObject(ptr, column.getSortOrder());
// We are guaranteed that the two column will have the same type.
if (!column
.getDataType()
.isSizeCompatible(
ptr,
value,
column.getDataType(),
expression.getMaxLength(),
expression.getScale(),
column.getMaxLength(),
column.getScale())) {
throw new ValueTypeIncompatibleException(
column.getDataType(), column.getMaxLength(), column.getScale());
}
column
.getDataType()
.coerceBytes(
ptr,
value,
expression.getDataType(),
expression.getMaxLength(),
expression.getScale(),
expression.getSortOrder(),
column.getMaxLength(),
column.getScale(),
column.getSortOrder());
byte[] bytes = ByteUtil.copyKeyBytesIfNecessary(ptr);
row.setValue(column, bytes);
}
}
for (Mutation mutation : row.toRowMutations()) {
mutations.add(mutation);
}
} else if (deleteCF != null && deleteCQ != null) {
// No need to search for delete column, since we project only it
// if no empty key value is being set
if (emptyCF == null || result.getValue(deleteCF, deleteCQ) != null) {
Delete delete =
new Delete(
results.get(0).getRowArray(),
results.get(0).getRowOffset(),
results.get(0).getRowLength());
delete.deleteColumns(deleteCF, deleteCQ, ts);
mutations.add(delete);
}
}
if (emptyCF != null) {
/*
* If we've specified an emptyCF, then we need to insert an empty
* key value "retroactively" for any key value that is visible at
* the timestamp that the DDL was issued. Key values that are not
* visible at this timestamp will not ever be projected up to
* scans past this timestamp, so don't need to be considered.
* We insert one empty key value per row per timestamp.
*/
Set<Long> timeStamps = Sets.newHashSetWithExpectedSize(results.size());
for (Cell kv : results) {
long kvts = kv.getTimestamp();
if (!timeStamps.contains(kvts)) {
Put put = new Put(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength());
put.add(
emptyCF, QueryConstants.EMPTY_COLUMN_BYTES, kvts, ByteUtil.EMPTY_BYTE_ARRAY);
mutations.add(put);
}
}
}
// Commit in batches based on UPSERT_BATCH_SIZE_ATTRIB in config
if (!mutations.isEmpty() && batchSize > 0 && mutations.size() % batchSize == 0) {
commitBatch(region, mutations, indexUUID);
mutations.clear();
}
} catch (ConstraintViolationException e) {
// Log and ignore in count
logger.error(
"Failed to create row in "
+ region.getRegionNameAsString()
+ " with values "
+ SchemaUtil.toString(values),
e);
continue;
}
aggregators.aggregate(rowAggregators, result);
hasAny = true;
}
} while (hasMore);
} finally {
innerScanner.close();
region.closeRegionOperation();
}
if (logger.isInfoEnabled()) {
logger.info("Finished scanning " + rowCount + " rows for ungrouped coprocessor scan " + scan);
}
if (!mutations.isEmpty()) {
commitBatch(region, mutations, indexUUID);
}
final boolean hadAny = hasAny;
KeyValue keyValue = null;
if (hadAny) {
byte[] value = aggregators.toBytes(rowAggregators);
keyValue =
KeyValueUtil.newKeyValue(
UNGROUPED_AGG_ROW_KEY,
SINGLE_COLUMN_FAMILY,
SINGLE_COLUMN,
AGG_TIMESTAMP,
value,
0,
value.length);
}
final KeyValue aggKeyValue = keyValue;
RegionScanner scanner =
new BaseRegionScanner() {
private boolean done = !hadAny;
@Override
public HRegionInfo getRegionInfo() {
return innerScanner.getRegionInfo();
}
@Override
public boolean isFilterDone() {
return done;
}
@Override
public void close() throws IOException {
innerScanner.close();
}
@Override
public boolean next(List<Cell> results) throws IOException {
if (done) return false;
done = true;
results.add(aggKeyValue);
return false;
}
@Override
public long getMaxResultSize() {
return scan.getMaxResultSize();
}
};
return scanner;
}