/**
* Encodes a String with digits into a "customized reversed packed binary coded decimal" byte
* array.
*/
void encodedCustomizedReversedPackedBcd(
String decimalDigits, ImmutableBytesWritable bytesWritable) {
byte[] bytes = bytesWritable.get();
int offset = bytesWritable.getOffset();
final int encodedLength = getSerializedLength(decimalDigits);
final char[] digits = decimalDigits.toCharArray();
for (int i = 0; i < encodedLength; i++) {
byte bcd = TWO_TERMINATOR_NIBBLES; // initialize with termination nibbles
int digitsIdx = 2 * i;
boolean firstNibbleWritten = false;
if (digitsIdx < digits.length) {
bcd = (byte) (lookupDigit(digits[digitsIdx]) << 4);
firstNibbleWritten = true;
}
if (++digitsIdx < digits.length) {
bcd |= lookupDigit(digits[digitsIdx]);
} else if (firstNibbleWritten) {
bcd |= TERMINATOR_NIBBLE; // uneven number of digits -> write terminator nibble
}
bytes[offset + i] = mask(bcd); // this could be two bcd nibbles or two terminator nibbles
}
RowKeyUtils.seek(bytesWritable, encodedLength);
}
@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
BigDecimal result = null;
DateNative dateNative = DateNative.getInstance();
for (int i = 0; i < children.size(); i++) {
if (!children.get(i).evaluate(tuple, ptr) || ptr.getLength() == 0) {
return false;
}
PDataType childType = children.get(i).getDataType();
boolean isDate = childType.isCoercibleTo(PDataType.DATE);
BigDecimal bd =
isDate
? BigDecimal.valueOf(dateNative.toLong(ptr))
: (BigDecimal) PDataType.DECIMAL.toObject(ptr, childType);
if (result == null) {
result = bd;
} else {
result = result.subtract(bd);
/*
* Special case for date subtraction - note that only first two expression may be dates.
* We need to convert the date to a unit of "days" because that's what sql expects.
*/
if (isDate) {
result = result.divide(BD_MILLIS_IN_DAY, PDataType.DEFAULT_MATH_CONTEXT);
}
}
}
ptr.set(PDataType.DECIMAL.toBytes(result));
return true;
}
@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
BigDecimal result = null;
for (int i = 0; i < children.size(); i++) {
Expression childExpr = children.get(i);
if (!childExpr.evaluate(tuple, ptr)) {
return false;
}
if (ptr.getLength() == 0) {
return true;
}
PDataType childType = children.get(i).getDataType();
BigDecimal bd = (BigDecimal) PDataType.DECIMAL.toObject(ptr, childType);
if (result == null) {
result = bd;
} else {
result = result.divide(bd, PDataType.DEFAULT_MATH_CONTEXT);
}
}
if (maxLength != null && scale != null) {
result = NumberUtil.setDecimalWidthAndScale(result, maxLength, scale);
}
if (result == null) {
throw new ValueTypeIncompatibleException(PDataType.DECIMAL, maxLength, scale);
}
ptr.set(PDataType.DECIMAL.toBytes(result));
return true;
}
@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
if (!children.get(0).evaluate(tuple, ptr)) {
return false;
}
String timezone = Bytes.toString(ptr.get(), ptr.getOffset(), ptr.getLength());
if (!children.get(1).evaluate(tuple, ptr)) {
return false;
}
if (!cachedTimeZones.containsKey(timezone)) {
TimeZone tz = TimeZone.getTimeZone(timezone);
if (!tz.getID().equals(timezone)) {
throw new IllegalDataException("Invalid timezone " + timezone);
}
cachedTimeZones.put(timezone, tz);
}
Date date = (Date) PDate.INSTANCE.toObject(ptr, children.get(1).getSortOrder());
int offset = cachedTimeZones.get(timezone).getOffset(date.getTime());
ptr.set(PInteger.INSTANCE.toBytes(offset / MILLIS_TO_MINUTES));
return true;
}
@Override
public MetaDataMutationResult updateIndexState(
List<Mutation> tableMetadata, String parentTableName) throws SQLException {
byte[][] rowKeyMetadata = new byte[3][];
SchemaUtil.getVarChars(tableMetadata.get(0).getRow(), rowKeyMetadata);
Mutation m = MetaDataUtil.getTableHeaderRow(tableMetadata);
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
if (!MetaDataUtil.getMutationValue(m, INDEX_STATE_BYTES, kvBuilder, ptr)) {
throw new IllegalStateException();
}
PIndexState newState = PIndexState.fromSerializedValue(ptr.get()[ptr.getOffset()]);
byte[] tenantIdBytes = rowKeyMetadata[PhoenixDatabaseMetaData.TENANT_ID_INDEX];
String schemaName = Bytes.toString(rowKeyMetadata[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX]);
String indexName = Bytes.toString(rowKeyMetadata[PhoenixDatabaseMetaData.TABLE_NAME_INDEX]);
String indexTableName = SchemaUtil.getTableName(schemaName, indexName);
PName tenantId = tenantIdBytes.length == 0 ? null : PNameFactory.newName(tenantIdBytes);
PTable index = metaData.getTableRef(new PTableKey(tenantId, indexTableName)).getTable();
index =
PTableImpl.makePTable(
index,
newState == PIndexState.USABLE
? PIndexState.ACTIVE
: newState == PIndexState.UNUSABLE ? PIndexState.INACTIVE : newState);
return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, 0, index);
}
@Override
public void reduce(ImmutableBytesWritable key, Iterable<FloatWritable> values, Context context)
throws IOException, InterruptedException {
float maxTemp = 999;
float minTemp = 999; // TODO : set these to appropriate initial values
/// TODO : iterate throuth values
/// extract temp, calculate max / min
int count = 0;
for (FloatWritable val : values) {
float temp = val.get();
/// TODO : calculate min, max
}
/// once we have min, max temps.. save them
Put put = new Put(key.get());
/// TODO : populate the Put with min, max temp
context.write(key, put);
/// debug
System.out.println(
String.format(
"sensor : %d, maxTemp : %f, minTemp : %f, record#count : %d",
Bytes.toInt(key.get()), maxTemp, minTemp, count));
}
private int setEndKey(ImmutableBytesWritable ptr, int offset, int i) {
int length = ptr.getOffset() - offset;
endKey = copyKey(endKey, length + this.maxKeyLength, ptr.get(), offset, length);
endKeyLength = length;
endKeyLength += setKey(Bound.UPPER, endKey, length, i);
return length;
}
/**
* Check that none of no columns in our updatable VIEW are changing values.
*
* @param tableRef
* @param overlapViewColumns
* @param targetColumns
* @param projector
* @throws SQLException
*/
private static void throwIfNotUpdatable(
TableRef tableRef,
Set<PColumn> overlapViewColumns,
List<PColumn> targetColumns,
RowProjector projector,
boolean sameTable)
throws SQLException {
PTable table = tableRef.getTable();
if (table.getViewType() == ViewType.UPDATABLE && !overlapViewColumns.isEmpty()) {
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
for (int i = 0; i < targetColumns.size(); i++) {
PColumn targetColumn = targetColumns.get(i);
if (overlapViewColumns.contains(targetColumn)) {
Expression source = projector.getColumnProjector(i).getExpression();
if (source.isStateless()) {
source.evaluate(null, ptr);
if (Bytes.compareTo(
ptr.get(),
ptr.getOffset(),
ptr.getLength(),
targetColumn.getViewConstant(),
0,
targetColumn.getViewConstant().length - 1)
== 0) {
continue;
}
}
throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_UPDATE_VIEW_COLUMN)
.setColumnName(targetColumn.getName().getString())
.build()
.buildException();
}
}
}
}
@Override
public void serialize(Object o, ImmutableBytesWritable bytesWritable) throws IOException {
byte[] bytesToWriteIn = bytesWritable.get();
int offset = bytesWritable.getOffset();
if (o == null) {
if (fixedPrefixLength > 0)
throw new IllegalStateException(
"excepted at least " + fixedPrefixLength + " bytes to write");
else if (terminate()) {
// write one (masked) null byte
bytesToWriteIn[offset] = mask(NULL);
RowKeyUtils.seek(bytesWritable, 1);
}
} else {
final BytesWritable input = (BytesWritable) o;
if (fixedPrefixLength > input.getLength())
throw new IllegalStateException(
"excepted at least " + fixedPrefixLength + " bytes to write");
else {
encodeFixedPrefix(input.getBytes(), bytesWritable);
encodedCustomizedReversedPackedBcd(
toStringRepresentation(
input.getBytes(), fixedPrefixLength, input.getLength() - fixedPrefixLength),
bytesWritable);
}
}
}
private void encodeFixedPrefix(byte[] input, ImmutableBytesWritable bytesWritable) {
final byte[] output = bytesWritable.get();
final int offset = bytesWritable.getOffset();
for (int i = 0; i < fixedPrefixLength; i++) {
output[offset + i] = mask(input[i]);
}
RowKeyUtils.seek(bytesWritable, fixedPrefixLength);
}
@Override
public void aggregate(Tuple tuple, ImmutableBytesWritable ptr) {
ImmutableBytesPtr key = new ImmutableBytesPtr(ptr.get(), ptr.getOffset(), ptr.getLength());
Integer count = this.valueVsCount.get(key);
if (count == null) {
this.valueVsCount.put(key, 1);
} else {
this.valueVsCount.put(key, count++);
}
}
@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
tuple.getKey(ptr);
int offset = accessor.getOffset(ptr.get(), ptr.getOffset());
// Null is represented in the last expression of a multi-part key
// by the bytes not being present.
if (offset < ptr.getOffset() + ptr.getLength()) {
byte[] buffer = ptr.get();
int maxByteSize = ptr.getLength() - (offset - ptr.getOffset());
int fixedByteSize = -1;
// FIXME: fixedByteSize <= maxByteSize ? fixedByteSize : 0 required because HBase passes bogus
// keys to filter to position scan (HBASE-6562)
if (fromType.isFixedWidth()) {
fixedByteSize = getByteSize();
fixedByteSize = fixedByteSize <= maxByteSize ? fixedByteSize : 0;
}
int length =
fixedByteSize >= 0 ? fixedByteSize : accessor.getLength(buffer, offset, maxByteSize);
// In the middle of the key, an empty variable length byte array represents null
if (length > 0) {
if (type == fromType) {
ptr.set(buffer, offset, length);
} else {
ptr.set(type.toBytes(type.toObject(buffer, offset, length, fromType)));
}
return true;
}
}
return false;
}
private void evaluateAndAssertResult(
Expression expression, Object expectedResult, String context) {
context = context == null ? "" : context;
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
assertTrue(expression.evaluate(null, ptr));
PDataType dataType = expression.getDataType();
SortOrder sortOrder = expression.getSortOrder();
Object result =
dataType.toObject(ptr.get(), ptr.getOffset(), ptr.getLength(), dataType, sortOrder);
assertEquals(context, expectedResult, result);
}
@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
if (!getChildExpression().evaluate(tuple, ptr)) {
return false;
}
// Update the digest value
messageDigest.update(ptr.get(), ptr.getOffset(), ptr.getLength());
// Get the digest bytes (note this resets the messageDigest as well)
ptr.set(messageDigest.digest());
return true;
}
private byte[] decodeFixedPrefix(ImmutableBytesWritable input) {
final byte[] output = new byte[fixedPrefixLength];
final byte[] inputBytes = input.get();
final int offset = input.getOffset();
for (int i = 0; i < fixedPrefixLength; i++) {
output[i] = mask(inputBytes[offset + i]);
}
RowKeyUtils.seek(input, fixedPrefixLength);
return output;
}
protected void reduce(ImmutableBytesWritable key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
// TODO Auto-generated method stub
Put put = new Put(key.get());
put.add(Bytes.toBytes("details"), Bytes.toBytes("total"), Bytes.toBytes(sum));
System.out.println(
String.format("stats : key : %d, count : %d", Bytes.toInt(key.get()), sum));
context.write(key, put);
}
@Override
public void skip(ImmutableBytesWritable bytesWritable) throws IOException {
if (bytesWritable.getLength() <= 0) return;
byte[] bytes = bytesWritable.get();
int offset = bytesWritable.getOffset();
int len = bytesWritable.getLength();
RowKeyUtils.seek(
bytesWritable,
fixedPrefixLength
+ getBcdEncodedLength(bytes, offset + fixedPrefixLength, len - fixedPrefixLength));
}
private int setStartKey(ImmutableBytesWritable ptr, int offset, int i) {
int length = ptr.getOffset() - offset;
startKey = copyKey(startKey, length + this.maxKeyLength, ptr.get(), offset, length);
startKeyLength = length;
// Add separator byte if we're at the end of the buffer, since trailing separator bytes are
// stripped
if (ptr.getOffset() + ptr.getLength() == offset + length
&& i - 1 > 0
&& !schema.getField(i - 1).getType().isFixedWidth()) {
startKey[startKeyLength++] = QueryConstants.SEPARATOR_BYTE;
}
startKeyLength += setKey(Bound.LOWER, startKey, startKeyLength, i);
return length;
}
/**
* Collect all column values for the same Row. RowKey may be different if indexes are involved, so
* it writes a separate record for each unique RowKey
*
* @param context Current mapper context
* @param tableName Table index in tableNames list
* @param lkv List of KV values that will be combined in a single ImmutableBytesWritable
* @throws IOException
* @throws InterruptedException
*/
private void writeAggregatedRow(Context context, String tableName, List<KeyValue> lkv)
throws IOException, InterruptedException {
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
DataOutputStream outputStream = new DataOutputStream(bos);
ImmutableBytesWritable outputKey = null;
if (!lkv.isEmpty()) {
for (KeyValue cell : lkv) {
if (outputKey == null
|| Bytes.compareTo(
outputKey.get(),
outputKey.getOffset(),
outputKey.getLength(),
cell.getRowArray(),
cell.getRowOffset(),
cell.getRowLength())
!= 0) {
// This a the first RowKey or a different from previous
if (outputKey != null) { // It's a different RowKey, so we need to write it
ImmutableBytesWritable aggregatedArray = new ImmutableBytesWritable(bos.toByteArray());
outputStream.close();
context.write(new TableRowkeyPair(tableName, outputKey), aggregatedArray);
}
outputKey =
new ImmutableBytesWritable(
cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
bos = new ByteArrayOutputStream(1024);
outputStream = new DataOutputStream(bos);
}
/*
The order of aggregation: type, index of column, length of value, value itself
*/
int i = findIndex(cell);
if (i == -1) {
// That may happen when we load only local indexes. Since KV pairs for both
// table and local index are going to the same physical table at that point
// we skip those KVs that are not belongs to loca index
continue;
}
outputStream.writeByte(cell.getTypeByte());
WritableUtils.writeVInt(outputStream, i);
WritableUtils.writeVInt(outputStream, cell.getValueLength());
outputStream.write(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
}
ImmutableBytesWritable aggregatedArray = new ImmutableBytesWritable(bos.toByteArray());
outputStream.close();
context.write(new TableRowkeyPair(tableName, outputKey), aggregatedArray);
}
}
private void iterate() {
if (schema.next(ptr, position++, maxOffset) == null) {
nextValue = null;
} else {
nextValue = ptr.copyBytes();
}
}
private static Put resultToPut(ImmutableBytesWritable key, Result result) throws IOException {
Put put = new Put(key.get());
for (Cell kv : result.rawCells()) {
put.add(kv);
}
return put;
}
// @Override
// protected void map(ImmutableBytesWritable key, Text value, Context context) throws
// IOException, InterruptedException {
// Text combinedKeyValue = new Text();
// //the structure is key###value
// combinedKeyValue.set(Bytes.toString(key.get()) + "###" + value.toString());
// context.write(one, combinedKeyValue);
// }
@Override
protected void map(ImmutableBytesWritable key, Result columns, Context context)
throws IOException, InterruptedException {
Text combinedKeyValue = new Text();
// the structure is key###value
String value = null;
try {
for (KeyValue kv : columns.list()) {
byte[] gmmData = kv.getValue();
String gmmString = Bytes.toStringBinary(gmmData);
// /* just for checking that gmm is correctly constructed
MixtureModel m = null;
m = (MixtureModel) ObjectAndByte.byteArrayToObject(Bytes.toBytesBinary(gmmString));
System.out.println("m.size:" + m.size);
// */
combinedKeyValue.set(Bytes.toString(key.get()) + "###" + gmmString);
context.write(one, combinedKeyValue);
// context.write(key, new Text(gmmString));
}
} catch (Exception e) {
e.printStackTrace();
}
}
/** @return The HTableDescriptor to create or alter the table */
protected HTableDescriptor getHTableDescriptor() {
HTableDescriptor hTableDescriptor = new HTableDescriptor(tableKey.get());
for (HColumnDescriptor family : this.columnFamilies) {
hTableDescriptor.addFamily(family);
}
return hTableDescriptor;
}
public int getPartition(ImmutableBytesWritable key, V2 value, int numPartitions) {
byte[] region = null;
// Only one region return 0
if (this.startKeys.length == 1) {
return 0;
}
try {
// Not sure if this is cached after a split so we could have problems
// here if a region splits while mapping
region = table.getRegionLocation(key.get()).getRegionInfo().getStartKey();
} catch (IOException e) {
LOG.error(e);
}
for (int i = 0; i < this.startKeys.length; i++) {
if (Bytes.compareTo(region, this.startKeys[i]) == 0) {
if (i >= numPartitions - 1) {
// cover if we have less reduces then regions.
return (Integer.toString(i).hashCode() & Integer.MAX_VALUE) % numPartitions;
}
return i;
}
}
// if above fails to find start key that match we need to return something
return 0;
}
public PRowImpl(ImmutableBytesWritable key, long ts) {
byte[] keyBytes = key.copyBytes();
this.setValues = new Put(keyBytes);
this.unsetValues = new Delete(keyBytes);
this.ts = ts;
this.key = keyBytes;
}
@Override
public final Object getValue(Tuple tuple, PDataType type, ImmutableBytesWritable ptr)
throws SQLException {
try {
Expression expression = getExpression();
if (!expression.evaluate(tuple, ptr)) {
return null;
}
if (ptr.getLength() == 0) {
return null;
}
return type.toObject(
ptr,
expression.getDataType(),
expression.getSortOrder(),
expression.getMaxLength(),
expression.getScale());
} catch (RuntimeException e) {
// FIXME: Expression.evaluate does not throw SQLException
// so this will unwrap throws from that.
if (e.getCause() instanceof SQLException) {
throw (SQLException) e.getCause();
}
throw e;
}
}
@Override
public KeyValue getValue(byte[] family, byte[] qualifier) {
if (keyValue == null) {
keyValue =
KeyValueUtil.newKeyValue(
keyPtr.get(),
keyPtr.getOffset(),
keyPtr.getLength(),
VALUE_COLUMN_FAMILY,
VALUE_COLUMN_QUALIFIER,
timestamp,
projectedValue,
0,
projectedValue.length);
}
return keyValue;
}
@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
// Starting from the end of the byte, look for all single bytes at the end of the string
// that is below SPACE_UTF8 (space and control characters) or above (control chars).
if (!getStringExpression().evaluate(tuple, ptr)) {
return false;
}
if (ptr.getLength() == 0) {
ptr.set(ByteUtil.EMPTY_BYTE_ARRAY);
return true;
}
byte[] string = ptr.get();
int offset = ptr.getOffset();
int length = ptr.getLength();
int i = offset + length - 1;
for (; i >= offset; i--) {
if ((string[i] & SINGLE_BYTE_MASK) == 0 && SPACE_UTF8 < string[i] && string[i] != 0x7f) {
break;
}
}
if (i == offset - 1) {
ptr.set(ByteUtil.EMPTY_BYTE_ARRAY);
return true;
}
ptr.set(string, offset, i - offset + 1);
return true;
}
@Override
public void map(ImmutableBytesWritable row, Result value, Context context)
throws InterruptedException, IOException {
KeyValue[] kvs = value.raw();
if (lastRowKey == null) {
lastRowKey = row.get();
} else if (Bytes.compareTo(lastRowKey, row.get()) != 0) {
writeLine(context, Bytes.toString(lastRowKey));
columnValueMap.clear();
}
for (KeyValue kv : kvs) {
String qualifier = Bytes.toString(kv.getQualifier());
byte[] val = kv.getValue();
columnValueMap.put(qualifier, val);
}
}
@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
Expression arg = getChildren().get(0);
if (!arg.evaluate(tuple, ptr)) {
return false;
}
if (ptr.getLength() == 0) {
return true;
}
long dateTime = inputCodec.decodeLong(ptr, arg.getSortOrder());
DateTime jodaDT = new DateTime(dateTime);
int day = jodaDT.getDayOfYear();
PDataType returnDataType = getDataType();
byte[] byteValue = new byte[returnDataType.getByteSize()];
returnDataType.getCodec().encodeInt(day, byteValue, 0);
ptr.set(byteValue);
return true;
}