@Override
public void reduce(ImmutableBytesWritable key, Iterable<FloatWritable> values, Context context)
throws IOException, InterruptedException {
float maxTemp = 999;
float minTemp = 999; // TODO : set these to appropriate initial values
/// TODO : iterate throuth values
/// extract temp, calculate max / min
int count = 0;
for (FloatWritable val : values) {
float temp = val.get();
/// TODO : calculate min, max
}
/// once we have min, max temps.. save them
Put put = new Put(key.get());
/// TODO : populate the Put with min, max temp
context.write(key, put);
/// debug
System.out.println(
String.format(
"sensor : %d, maxTemp : %f, minTemp : %f, record#count : %d",
Bytes.toInt(key.get()), maxTemp, minTemp, count));
}
@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
tuple.getKey(ptr);
int offset = accessor.getOffset(ptr.get(), ptr.getOffset());
// Null is represented in the last expression of a multi-part key
// by the bytes not being present.
if (offset < ptr.getOffset() + ptr.getLength()) {
byte[] buffer = ptr.get();
int maxByteSize = ptr.getLength() - (offset - ptr.getOffset());
int fixedByteSize = -1;
// FIXME: fixedByteSize <= maxByteSize ? fixedByteSize : 0 required because HBase passes bogus
// keys to filter to position scan (HBASE-6562)
if (fromType.isFixedWidth()) {
fixedByteSize = getByteSize();
fixedByteSize = fixedByteSize <= maxByteSize ? fixedByteSize : 0;
}
int length =
fixedByteSize >= 0 ? fixedByteSize : accessor.getLength(buffer, offset, maxByteSize);
// In the middle of the key, an empty variable length byte array represents null
if (length > 0) {
if (type == fromType) {
ptr.set(buffer, offset, length);
} else {
ptr.set(type.toBytes(type.toObject(buffer, offset, length, fromType)));
}
return true;
}
}
return false;
}
@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
// TODO: multi-byte characters
Expression offsetExpression = getOffsetExpression();
if (!offsetExpression.evaluate(tuple, ptr)) {
return false;
}
LongNative longNative = (LongNative) offsetExpression.getDataType().getNative();
int offset = (int) longNative.toLong(ptr);
int length = -1;
if (hasLengthExpression) {
Expression lengthExpression = getLengthExpression();
if (!lengthExpression.evaluate(tuple, ptr)) {
return false;
}
longNative = (LongNative) lengthExpression.getDataType().getNative();
length = (int) longNative.toLong(ptr);
if (length <= 0) {
return false;
}
}
if (!getStrExpression().evaluate(tuple, ptr)) {
return false;
}
try {
boolean isCharType = getStrExpression().getDataType() == PDataType.CHAR;
int strlen =
isCharType
? ptr.getLength()
: StringUtil.calculateUTF8Length(ptr.get(), ptr.getOffset(), ptr.getLength());
// Account for 1 versus 0-based offset
offset = offset - (offset <= 0 ? 0 : 1);
if (offset < 0) { // Offset < 0 means get from end
offset = strlen + offset;
}
if (offset < 0 || offset >= strlen) {
return false;
}
int maxLength = strlen - offset;
length = length == -1 ? maxLength : Math.min(length, maxLength);
int byteOffset =
isCharType
? offset
: StringUtil.getByteLengthForUtf8SubStr(ptr.get(), ptr.getOffset(), offset);
int byteLength =
isCharType
? length
: StringUtil.getByteLengthForUtf8SubStr(
ptr.get(), ptr.getOffset() + byteOffset, length);
ptr.set(ptr.get(), ptr.getOffset() + byteOffset, byteLength);
return true;
} catch (UnsupportedEncodingException e) {
return false;
}
}
public static ProjectedValueTuple mergeProjectedValue(
ProjectedValueTuple dest,
KeyValueSchema destSchema,
ValueBitSet destBitSet,
Tuple src,
KeyValueSchema srcSchema,
ValueBitSet srcBitSet,
int offset)
throws IOException {
ImmutableBytesWritable destValue = new ImmutableBytesWritable(dest.getProjectedValue());
destBitSet.clear();
destBitSet.or(destValue);
int origDestBitSetLen = dest.getBitSetLength();
ImmutableBytesWritable srcValue = new ImmutableBytesWritable();
decodeProjectedValue(src, srcValue);
srcBitSet.clear();
srcBitSet.or(srcValue);
int origSrcBitSetLen = srcBitSet.getEstimatedLength();
for (int i = 0; i < srcBitSet.getMaxSetBit(); i++) {
if (srcBitSet.get(i)) {
destBitSet.set(offset + i);
}
}
int destBitSetLen = destBitSet.getEstimatedLength();
byte[] merged =
new byte
[destValue.getLength()
- origDestBitSetLen
+ srcValue.getLength()
- origSrcBitSetLen
+ destBitSetLen];
int o =
Bytes.putBytes(
merged,
0,
destValue.get(),
destValue.getOffset(),
destValue.getLength() - origDestBitSetLen);
o =
Bytes.putBytes(
merged,
o,
srcValue.get(),
srcValue.getOffset(),
srcValue.getLength() - origSrcBitSetLen);
destBitSet.toBytes(merged, o);
ImmutableBytesWritable keyPtr = dest.getKeyPtr();
return new ProjectedValueTuple(
keyPtr.get(),
keyPtr.getOffset(),
keyPtr.getLength(),
dest.getTimestamp(),
merged,
destBitSetLen);
}
protected void reduce(ImmutableBytesWritable key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
// TODO Auto-generated method stub
Put put = new Put(key.get());
put.add(Bytes.toBytes("details"), Bytes.toBytes("total"), Bytes.toBytes(sum));
System.out.println(
String.format("stats : key : %d, count : %d", Bytes.toInt(key.get()), sum));
context.write(key, put);
}
/**
* Check that none of no columns in our updatable VIEW are changing values.
*
* @param tableRef
* @param overlapViewColumns
* @param targetColumns
* @param projector
* @throws SQLException
*/
private static void throwIfNotUpdatable(
TableRef tableRef,
Set<PColumn> overlapViewColumns,
List<PColumn> targetColumns,
RowProjector projector,
boolean sameTable)
throws SQLException {
PTable table = tableRef.getTable();
if (table.getViewType() == ViewType.UPDATABLE && !overlapViewColumns.isEmpty()) {
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
for (int i = 0; i < targetColumns.size(); i++) {
PColumn targetColumn = targetColumns.get(i);
if (overlapViewColumns.contains(targetColumn)) {
Expression source = projector.getColumnProjector(i).getExpression();
if (source.isStateless()) {
source.evaluate(null, ptr);
if (Bytes.compareTo(
ptr.get(),
ptr.getOffset(),
ptr.getLength(),
targetColumn.getViewConstant(),
0,
targetColumn.getViewConstant().length - 1)
== 0) {
continue;
}
}
throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_UPDATE_VIEW_COLUMN)
.setColumnName(targetColumn.getName().getString())
.build()
.buildException();
}
}
}
}
// @Override
// protected void map(ImmutableBytesWritable key, Text value, Context context) throws
// IOException, InterruptedException {
// Text combinedKeyValue = new Text();
// //the structure is key###value
// combinedKeyValue.set(Bytes.toString(key.get()) + "###" + value.toString());
// context.write(one, combinedKeyValue);
// }
@Override
protected void map(ImmutableBytesWritable key, Result columns, Context context)
throws IOException, InterruptedException {
Text combinedKeyValue = new Text();
// the structure is key###value
String value = null;
try {
for (KeyValue kv : columns.list()) {
byte[] gmmData = kv.getValue();
String gmmString = Bytes.toStringBinary(gmmData);
// /* just for checking that gmm is correctly constructed
MixtureModel m = null;
m = (MixtureModel) ObjectAndByte.byteArrayToObject(Bytes.toBytesBinary(gmmString));
System.out.println("m.size:" + m.size);
// */
combinedKeyValue.set(Bytes.toString(key.get()) + "###" + gmmString);
context.write(one, combinedKeyValue);
// context.write(key, new Text(gmmString));
}
} catch (Exception e) {
e.printStackTrace();
}
}
@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
if (!children.get(0).evaluate(tuple, ptr)) {
return false;
}
String timezone = Bytes.toString(ptr.get(), ptr.getOffset(), ptr.getLength());
if (!children.get(1).evaluate(tuple, ptr)) {
return false;
}
if (!cachedTimeZones.containsKey(timezone)) {
TimeZone tz = TimeZone.getTimeZone(timezone);
if (!tz.getID().equals(timezone)) {
throw new IllegalDataException("Invalid timezone " + timezone);
}
cachedTimeZones.put(timezone, tz);
}
Date date = (Date) PDate.INSTANCE.toObject(ptr, children.get(1).getSortOrder());
int offset = cachedTimeZones.get(timezone).getOffset(date.getTime());
ptr.set(PInteger.INSTANCE.toBytes(offset / MILLIS_TO_MINUTES));
return true;
}
@Override
public MetaDataMutationResult updateIndexState(
List<Mutation> tableMetadata, String parentTableName) throws SQLException {
byte[][] rowKeyMetadata = new byte[3][];
SchemaUtil.getVarChars(tableMetadata.get(0).getRow(), rowKeyMetadata);
Mutation m = MetaDataUtil.getTableHeaderRow(tableMetadata);
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
if (!MetaDataUtil.getMutationValue(m, INDEX_STATE_BYTES, kvBuilder, ptr)) {
throw new IllegalStateException();
}
PIndexState newState = PIndexState.fromSerializedValue(ptr.get()[ptr.getOffset()]);
byte[] tenantIdBytes = rowKeyMetadata[PhoenixDatabaseMetaData.TENANT_ID_INDEX];
String schemaName = Bytes.toString(rowKeyMetadata[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX]);
String indexName = Bytes.toString(rowKeyMetadata[PhoenixDatabaseMetaData.TABLE_NAME_INDEX]);
String indexTableName = SchemaUtil.getTableName(schemaName, indexName);
PName tenantId = tenantIdBytes.length == 0 ? null : PNameFactory.newName(tenantIdBytes);
PTable index = metaData.getTableRef(new PTableKey(tenantId, indexTableName)).getTable();
index =
PTableImpl.makePTable(
index,
newState == PIndexState.USABLE
? PIndexState.ACTIVE
: newState == PIndexState.UNUSABLE ? PIndexState.INACTIVE : newState);
return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, 0, index);
}
/**
* Encodes a String with digits into a "customized reversed packed binary coded decimal" byte
* array.
*/
void encodedCustomizedReversedPackedBcd(
String decimalDigits, ImmutableBytesWritable bytesWritable) {
byte[] bytes = bytesWritable.get();
int offset = bytesWritable.getOffset();
final int encodedLength = getSerializedLength(decimalDigits);
final char[] digits = decimalDigits.toCharArray();
for (int i = 0; i < encodedLength; i++) {
byte bcd = TWO_TERMINATOR_NIBBLES; // initialize with termination nibbles
int digitsIdx = 2 * i;
boolean firstNibbleWritten = false;
if (digitsIdx < digits.length) {
bcd = (byte) (lookupDigit(digits[digitsIdx]) << 4);
firstNibbleWritten = true;
}
if (++digitsIdx < digits.length) {
bcd |= lookupDigit(digits[digitsIdx]);
} else if (firstNibbleWritten) {
bcd |= TERMINATOR_NIBBLE; // uneven number of digits -> write terminator nibble
}
bytes[offset + i] = mask(bcd); // this could be two bcd nibbles or two terminator nibbles
}
RowKeyUtils.seek(bytesWritable, encodedLength);
}
public int getPartition(ImmutableBytesWritable key, V2 value, int numPartitions) {
byte[] region = null;
// Only one region return 0
if (this.startKeys.length == 1) {
return 0;
}
try {
// Not sure if this is cached after a split so we could have problems
// here if a region splits while mapping
region = table.getRegionLocation(key.get()).getRegionInfo().getStartKey();
} catch (IOException e) {
LOG.error(e);
}
for (int i = 0; i < this.startKeys.length; i++) {
if (Bytes.compareTo(region, this.startKeys[i]) == 0) {
if (i >= numPartitions - 1) {
// cover if we have less reduces then regions.
return (Integer.toString(i).hashCode() & Integer.MAX_VALUE) % numPartitions;
}
return i;
}
}
// if above fails to find start key that match we need to return something
return 0;
}
/** @return The HTableDescriptor to create or alter the table */
protected HTableDescriptor getHTableDescriptor() {
HTableDescriptor hTableDescriptor = new HTableDescriptor(tableKey.get());
for (HColumnDescriptor family : this.columnFamilies) {
hTableDescriptor.addFamily(family);
}
return hTableDescriptor;
}
private static Put resultToPut(ImmutableBytesWritable key, Result result) throws IOException {
Put put = new Put(key.get());
for (Cell kv : result.rawCells()) {
put.add(kv);
}
return put;
}
@Override
public void serialize(Object o, ImmutableBytesWritable bytesWritable) throws IOException {
byte[] bytesToWriteIn = bytesWritable.get();
int offset = bytesWritable.getOffset();
if (o == null) {
if (fixedPrefixLength > 0)
throw new IllegalStateException(
"excepted at least " + fixedPrefixLength + " bytes to write");
else if (terminate()) {
// write one (masked) null byte
bytesToWriteIn[offset] = mask(NULL);
RowKeyUtils.seek(bytesWritable, 1);
}
} else {
final BytesWritable input = (BytesWritable) o;
if (fixedPrefixLength > input.getLength())
throw new IllegalStateException(
"excepted at least " + fixedPrefixLength + " bytes to write");
else {
encodeFixedPrefix(input.getBytes(), bytesWritable);
encodedCustomizedReversedPackedBcd(
toStringRepresentation(
input.getBytes(), fixedPrefixLength, input.getLength() - fixedPrefixLength),
bytesWritable);
}
}
}
private int setEndKey(ImmutableBytesWritable ptr, int offset, int i) {
int length = ptr.getOffset() - offset;
endKey = copyKey(endKey, length + this.maxKeyLength, ptr.get(), offset, length);
endKeyLength = length;
endKeyLength += setKey(Bound.UPPER, endKey, length, i);
return length;
}
@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
// Starting from the end of the byte, look for all single bytes at the end of the string
// that is below SPACE_UTF8 (space and control characters) or above (control chars).
if (!getStringExpression().evaluate(tuple, ptr)) {
return false;
}
if (ptr.getLength() == 0) {
ptr.set(ByteUtil.EMPTY_BYTE_ARRAY);
return true;
}
byte[] string = ptr.get();
int offset = ptr.getOffset();
int length = ptr.getLength();
int i = offset + length - 1;
for (; i >= offset; i--) {
if ((string[i] & SINGLE_BYTE_MASK) == 0 && SPACE_UTF8 < string[i] && string[i] != 0x7f) {
break;
}
}
if (i == offset - 1) {
ptr.set(ByteUtil.EMPTY_BYTE_ARRAY);
return true;
}
ptr.set(string, offset, i - offset + 1);
return true;
}
private void encodeFixedPrefix(byte[] input, ImmutableBytesWritable bytesWritable) {
final byte[] output = bytesWritable.get();
final int offset = bytesWritable.getOffset();
for (int i = 0; i < fixedPrefixLength; i++) {
output[offset + i] = mask(input[i]);
}
RowKeyUtils.seek(bytesWritable, fixedPrefixLength);
}
@Override
public void map(ImmutableBytesWritable row, Result value, Context context)
throws InterruptedException, IOException {
KeyValue[] kvs = value.raw();
if (lastRowKey == null) {
lastRowKey = row.get();
} else if (Bytes.compareTo(lastRowKey, row.get()) != 0) {
writeLine(context, Bytes.toString(lastRowKey));
columnValueMap.clear();
}
for (KeyValue kv : kvs) {
String qualifier = Bytes.toString(kv.getQualifier());
byte[] val = kv.getValue();
columnValueMap.put(qualifier, val);
}
}
@Override
protected void map(ImmutableBytesWritable key, Result value, Context context)
throws IOException, InterruptedException {
HashMap<String, String> json = new HashMap<>();
json.put(ID_FIELD, new String(key.get()));
json.put(
TEXT_FIELD,
new String(value.getValue(BulkLoad.HBASE_COL_FAMILY, BulkLoad.HBASE_COL_NAME)));
context.write(NullWritable.get(), new Text(mapper.writeValueAsBytes(json)));
}
@Override
public void aggregate(Tuple tuple, ImmutableBytesWritable ptr) {
ImmutableBytesPtr key = new ImmutableBytesPtr(ptr.get(), ptr.getOffset(), ptr.getLength());
Integer count = this.valueVsCount.get(key);
if (count == null) {
this.valueVsCount.put(key, 1);
} else {
this.valueVsCount.put(key, count++);
}
}
@Override
protected void map(ImmutableBytesWritable rowKey, Result result, Context context)
throws IOException, InterruptedException {
long recordSize = GeneralHelpers.getHBaseResultKVSize(result);
long colCount = result.size();
String rowKeyStr = null;
if (rowKeyType == RowKeyType.STRING) {
rowKeyStr = Bytes.toString(rowKey.get());
} else if (rowKeyType == RowKeyType.USER_ID) {
rowKeyStr = TruthyHelpers.getUserIDStrFromBytes(rowKey.get());
} else {
if (this.useBigInt) {
rowKeyStr = TruthyHelpers.getTweetIDStrFromBigIntBytes(rowKey.get());
} else {
rowKeyStr = TruthyHelpers.getTweetIDStrFromBytes(rowKey.get());
}
}
if (lastRowKeyStr == null) {
// start counting for the first row key
lastRowKeyStr = rowKeyStr;
totalColCount = colCount;
totalRecordSize = recordSize;
} else {
if (lastRowKeyStr.equals(rowKeyStr)) {
// keep counting for the same row key
totalColCount += colCount;
totalRecordSize += recordSize;
} else {
// write information about last row key
StringBuffer sb = new StringBuffer();
sb.append(totalColCount).append('\t');
sb.append(totalRecordSize);
context.write(new Text(lastRowKeyStr), new Text(sb.toString()));
// start counting for current row key
lastRowKeyStr = rowKeyStr;
totalColCount = colCount;
totalRecordSize = recordSize;
}
}
}
@Override
protected void map(ImmutableBytesWritable row, Result value, Context context)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
ImmutableBytesWritable userKey = new ImmutableBytesWritable(row.get(), 0, Bytes.SIZEOF_INT);
context.write(userKey, one);
numRecords++;
if ((numRecords % 1000) == 0) {
context.setStatus("mapper processed" + numRecords + " records so far");
}
}
private void evaluateAndAssertResult(
Expression expression, Object expectedResult, String context) {
context = context == null ? "" : context;
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
assertTrue(expression.evaluate(null, ptr));
PDataType dataType = expression.getDataType();
SortOrder sortOrder = expression.getSortOrder();
Object result =
dataType.toObject(ptr.get(), ptr.getOffset(), ptr.getLength(), dataType, sortOrder);
assertEquals(context, expectedResult, result);
}
private byte[] decodeFixedPrefix(ImmutableBytesWritable input) {
final byte[] output = new byte[fixedPrefixLength];
final byte[] inputBytes = input.get();
final int offset = input.getOffset();
for (int i = 0; i < fixedPrefixLength; i++) {
output[i] = mask(inputBytes[offset + i]);
}
RowKeyUtils.seek(input, fixedPrefixLength);
return output;
}
public void map(ImmutableBytesWritable rowKey, Result columns, Context context)
throws IOException, InterruptedException {
try {
String oKey = new String(rowKey.get());
byte[] bprice = columns.getValue(Bytes.toBytes("stock"), Bytes.toBytes("return"));
String sprice = new String(bprice);
String ss = sprice + "!" + oKey;
context.write(new Text("A"), new Text(ss));
} catch (RuntimeException e) {
e.printStackTrace();
}
}
@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
if (!getChildExpression().evaluate(tuple, ptr)) {
return false;
}
// Update the digest value
messageDigest.update(ptr.get(), ptr.getOffset(), ptr.getLength());
// Get the digest bytes (note this resets the messageDigest as well)
ptr.set(messageDigest.digest());
return true;
}
@Override
public void skip(ImmutableBytesWritable bytesWritable) throws IOException {
if (bytesWritable.getLength() <= 0) return;
byte[] bytes = bytesWritable.get();
int offset = bytesWritable.getOffset();
int len = bytesWritable.getLength();
RowKeyUtils.seek(
bytesWritable,
fixedPrefixLength
+ getBcdEncodedLength(bytes, offset + fixedPrefixLength, len - fixedPrefixLength));
}
/**
* Decodes a "customized reversed packed binary coded decimal" byte array into a String with
* digits.
*
* @param bytesWritable the customized packed BCD encoded byte array
* @return The decoded value
*/
String decodeCustomizedReversedPackedBcd(
ImmutableBytesWritable bytesWritable, int offset, int length) {
int i = 0;
final byte[] bytes = bytesWritable.get();
StringBuilder sb = new StringBuilder();
while (i < length) {
byte c = mask(bytes[offset + i++]);
if (addDigit((byte) ((c >>> 4) & 0x0f), sb) || addDigit((byte) (c & 0x0f), sb)) break;
}
RowKeyUtils.seek(bytesWritable, i);
return sb.toString();
}
private int setStartKey(ImmutableBytesWritable ptr, int offset, int i) {
int length = ptr.getOffset() - offset;
startKey = copyKey(startKey, length + this.maxKeyLength, ptr.get(), offset, length);
startKeyLength = length;
// Add separator byte if we're at the end of the buffer, since trailing separator bytes are
// stripped
if (ptr.getOffset() + ptr.getLength() == offset + length
&& i - 1 > 0
&& !schema.getField(i - 1).getType().isFixedWidth()) {
startKey[startKeyLength++] = QueryConstants.SEPARATOR_BYTE;
}
startKeyLength += setKey(Bound.LOWER, startKey, startKeyLength, i);
return length;
}
@Override
protected void map(ImmutableBytesWritable rowKey, Result result, Context context)
throws IOException, InterruptedException {
List<KeyValue> kvList = result.list();
// boolean isTextIndex = tableName.equals(Constants.TEXTS_INDEX_TABLE_NAME);
long totalFreq = 0;
Iterator<KeyValue> iter = kvList.iterator();
while (iter.hasNext()) {
byte[] value = iter.next().getValue();
totalFreq += Bytes.toInt(value);
}
Text term = new Text(Bytes.toString(rowKey.get()));
context.write(term, new LongWritable(totalFreq));
}