/** * Encodes a String with digits into a "customized reversed packed binary coded decimal" byte * array. */ void encodedCustomizedReversedPackedBcd( String decimalDigits, ImmutableBytesWritable bytesWritable) { byte[] bytes = bytesWritable.get(); int offset = bytesWritable.getOffset(); final int encodedLength = getSerializedLength(decimalDigits); final char[] digits = decimalDigits.toCharArray(); for (int i = 0; i < encodedLength; i++) { byte bcd = TWO_TERMINATOR_NIBBLES; // initialize with termination nibbles int digitsIdx = 2 * i; boolean firstNibbleWritten = false; if (digitsIdx < digits.length) { bcd = (byte) (lookupDigit(digits[digitsIdx]) << 4); firstNibbleWritten = true; } if (++digitsIdx < digits.length) { bcd |= lookupDigit(digits[digitsIdx]); } else if (firstNibbleWritten) { bcd |= TERMINATOR_NIBBLE; // uneven number of digits -> write terminator nibble } bytes[offset + i] = mask(bcd); // this could be two bcd nibbles or two terminator nibbles } RowKeyUtils.seek(bytesWritable, encodedLength); }
@Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { BigDecimal result = null; DateNative dateNative = DateNative.getInstance(); for (int i = 0; i < children.size(); i++) { if (!children.get(i).evaluate(tuple, ptr) || ptr.getLength() == 0) { return false; } PDataType childType = children.get(i).getDataType(); boolean isDate = childType.isCoercibleTo(PDataType.DATE); BigDecimal bd = isDate ? BigDecimal.valueOf(dateNative.toLong(ptr)) : (BigDecimal) PDataType.DECIMAL.toObject(ptr, childType); if (result == null) { result = bd; } else { result = result.subtract(bd); /* * Special case for date subtraction - note that only first two expression may be dates. * We need to convert the date to a unit of "days" because that's what sql expects. */ if (isDate) { result = result.divide(BD_MILLIS_IN_DAY, PDataType.DEFAULT_MATH_CONTEXT); } } } ptr.set(PDataType.DECIMAL.toBytes(result)); return true; }
@Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { BigDecimal result = null; for (int i = 0; i < children.size(); i++) { Expression childExpr = children.get(i); if (!childExpr.evaluate(tuple, ptr)) { return false; } if (ptr.getLength() == 0) { return true; } PDataType childType = children.get(i).getDataType(); BigDecimal bd = (BigDecimal) PDataType.DECIMAL.toObject(ptr, childType); if (result == null) { result = bd; } else { result = result.divide(bd, PDataType.DEFAULT_MATH_CONTEXT); } } if (maxLength != null && scale != null) { result = NumberUtil.setDecimalWidthAndScale(result, maxLength, scale); } if (result == null) { throw new ValueTypeIncompatibleException(PDataType.DECIMAL, maxLength, scale); } ptr.set(PDataType.DECIMAL.toBytes(result)); return true; }
@Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { if (!children.get(0).evaluate(tuple, ptr)) { return false; } String timezone = Bytes.toString(ptr.get(), ptr.getOffset(), ptr.getLength()); if (!children.get(1).evaluate(tuple, ptr)) { return false; } if (!cachedTimeZones.containsKey(timezone)) { TimeZone tz = TimeZone.getTimeZone(timezone); if (!tz.getID().equals(timezone)) { throw new IllegalDataException("Invalid timezone " + timezone); } cachedTimeZones.put(timezone, tz); } Date date = (Date) PDate.INSTANCE.toObject(ptr, children.get(1).getSortOrder()); int offset = cachedTimeZones.get(timezone).getOffset(date.getTime()); ptr.set(PInteger.INSTANCE.toBytes(offset / MILLIS_TO_MINUTES)); return true; }
@Override public MetaDataMutationResult updateIndexState( List<Mutation> tableMetadata, String parentTableName) throws SQLException { byte[][] rowKeyMetadata = new byte[3][]; SchemaUtil.getVarChars(tableMetadata.get(0).getRow(), rowKeyMetadata); Mutation m = MetaDataUtil.getTableHeaderRow(tableMetadata); ImmutableBytesWritable ptr = new ImmutableBytesWritable(); if (!MetaDataUtil.getMutationValue(m, INDEX_STATE_BYTES, kvBuilder, ptr)) { throw new IllegalStateException(); } PIndexState newState = PIndexState.fromSerializedValue(ptr.get()[ptr.getOffset()]); byte[] tenantIdBytes = rowKeyMetadata[PhoenixDatabaseMetaData.TENANT_ID_INDEX]; String schemaName = Bytes.toString(rowKeyMetadata[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX]); String indexName = Bytes.toString(rowKeyMetadata[PhoenixDatabaseMetaData.TABLE_NAME_INDEX]); String indexTableName = SchemaUtil.getTableName(schemaName, indexName); PName tenantId = tenantIdBytes.length == 0 ? null : PNameFactory.newName(tenantIdBytes); PTable index = metaData.getTableRef(new PTableKey(tenantId, indexTableName)).getTable(); index = PTableImpl.makePTable( index, newState == PIndexState.USABLE ? PIndexState.ACTIVE : newState == PIndexState.UNUSABLE ? PIndexState.INACTIVE : newState); return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, 0, index); }
@Override public void reduce(ImmutableBytesWritable key, Iterable<FloatWritable> values, Context context) throws IOException, InterruptedException { float maxTemp = 999; float minTemp = 999; // TODO : set these to appropriate initial values /// TODO : iterate throuth values /// extract temp, calculate max / min int count = 0; for (FloatWritable val : values) { float temp = val.get(); /// TODO : calculate min, max } /// once we have min, max temps.. save them Put put = new Put(key.get()); /// TODO : populate the Put with min, max temp context.write(key, put); /// debug System.out.println( String.format( "sensor : %d, maxTemp : %f, minTemp : %f, record#count : %d", Bytes.toInt(key.get()), maxTemp, minTemp, count)); }
private int setEndKey(ImmutableBytesWritable ptr, int offset, int i) { int length = ptr.getOffset() - offset; endKey = copyKey(endKey, length + this.maxKeyLength, ptr.get(), offset, length); endKeyLength = length; endKeyLength += setKey(Bound.UPPER, endKey, length, i); return length; }
/** * Check that none of no columns in our updatable VIEW are changing values. * * @param tableRef * @param overlapViewColumns * @param targetColumns * @param projector * @throws SQLException */ private static void throwIfNotUpdatable( TableRef tableRef, Set<PColumn> overlapViewColumns, List<PColumn> targetColumns, RowProjector projector, boolean sameTable) throws SQLException { PTable table = tableRef.getTable(); if (table.getViewType() == ViewType.UPDATABLE && !overlapViewColumns.isEmpty()) { ImmutableBytesWritable ptr = new ImmutableBytesWritable(); for (int i = 0; i < targetColumns.size(); i++) { PColumn targetColumn = targetColumns.get(i); if (overlapViewColumns.contains(targetColumn)) { Expression source = projector.getColumnProjector(i).getExpression(); if (source.isStateless()) { source.evaluate(null, ptr); if (Bytes.compareTo( ptr.get(), ptr.getOffset(), ptr.getLength(), targetColumn.getViewConstant(), 0, targetColumn.getViewConstant().length - 1) == 0) { continue; } } throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_UPDATE_VIEW_COLUMN) .setColumnName(targetColumn.getName().getString()) .build() .buildException(); } } } }
@Override public void serialize(Object o, ImmutableBytesWritable bytesWritable) throws IOException { byte[] bytesToWriteIn = bytesWritable.get(); int offset = bytesWritable.getOffset(); if (o == null) { if (fixedPrefixLength > 0) throw new IllegalStateException( "excepted at least " + fixedPrefixLength + " bytes to write"); else if (terminate()) { // write one (masked) null byte bytesToWriteIn[offset] = mask(NULL); RowKeyUtils.seek(bytesWritable, 1); } } else { final BytesWritable input = (BytesWritable) o; if (fixedPrefixLength > input.getLength()) throw new IllegalStateException( "excepted at least " + fixedPrefixLength + " bytes to write"); else { encodeFixedPrefix(input.getBytes(), bytesWritable); encodedCustomizedReversedPackedBcd( toStringRepresentation( input.getBytes(), fixedPrefixLength, input.getLength() - fixedPrefixLength), bytesWritable); } } }
private void encodeFixedPrefix(byte[] input, ImmutableBytesWritable bytesWritable) { final byte[] output = bytesWritable.get(); final int offset = bytesWritable.getOffset(); for (int i = 0; i < fixedPrefixLength; i++) { output[offset + i] = mask(input[i]); } RowKeyUtils.seek(bytesWritable, fixedPrefixLength); }
@Override public void aggregate(Tuple tuple, ImmutableBytesWritable ptr) { ImmutableBytesPtr key = new ImmutableBytesPtr(ptr.get(), ptr.getOffset(), ptr.getLength()); Integer count = this.valueVsCount.get(key); if (count == null) { this.valueVsCount.put(key, 1); } else { this.valueVsCount.put(key, count++); } }
@Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { tuple.getKey(ptr); int offset = accessor.getOffset(ptr.get(), ptr.getOffset()); // Null is represented in the last expression of a multi-part key // by the bytes not being present. if (offset < ptr.getOffset() + ptr.getLength()) { byte[] buffer = ptr.get(); int maxByteSize = ptr.getLength() - (offset - ptr.getOffset()); int fixedByteSize = -1; // FIXME: fixedByteSize <= maxByteSize ? fixedByteSize : 0 required because HBase passes bogus // keys to filter to position scan (HBASE-6562) if (fromType.isFixedWidth()) { fixedByteSize = getByteSize(); fixedByteSize = fixedByteSize <= maxByteSize ? fixedByteSize : 0; } int length = fixedByteSize >= 0 ? fixedByteSize : accessor.getLength(buffer, offset, maxByteSize); // In the middle of the key, an empty variable length byte array represents null if (length > 0) { if (type == fromType) { ptr.set(buffer, offset, length); } else { ptr.set(type.toBytes(type.toObject(buffer, offset, length, fromType))); } return true; } } return false; }
private void evaluateAndAssertResult( Expression expression, Object expectedResult, String context) { context = context == null ? "" : context; ImmutableBytesWritable ptr = new ImmutableBytesWritable(); assertTrue(expression.evaluate(null, ptr)); PDataType dataType = expression.getDataType(); SortOrder sortOrder = expression.getSortOrder(); Object result = dataType.toObject(ptr.get(), ptr.getOffset(), ptr.getLength(), dataType, sortOrder); assertEquals(context, expectedResult, result); }
@Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { if (!getChildExpression().evaluate(tuple, ptr)) { return false; } // Update the digest value messageDigest.update(ptr.get(), ptr.getOffset(), ptr.getLength()); // Get the digest bytes (note this resets the messageDigest as well) ptr.set(messageDigest.digest()); return true; }
private byte[] decodeFixedPrefix(ImmutableBytesWritable input) { final byte[] output = new byte[fixedPrefixLength]; final byte[] inputBytes = input.get(); final int offset = input.getOffset(); for (int i = 0; i < fixedPrefixLength; i++) { output[i] = mask(inputBytes[offset + i]); } RowKeyUtils.seek(input, fixedPrefixLength); return output; }
protected void reduce(ImmutableBytesWritable key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { int sum = 0; for (IntWritable val : values) { sum += val.get(); } // TODO Auto-generated method stub Put put = new Put(key.get()); put.add(Bytes.toBytes("details"), Bytes.toBytes("total"), Bytes.toBytes(sum)); System.out.println( String.format("stats : key : %d, count : %d", Bytes.toInt(key.get()), sum)); context.write(key, put); }
@Override public void skip(ImmutableBytesWritable bytesWritable) throws IOException { if (bytesWritable.getLength() <= 0) return; byte[] bytes = bytesWritable.get(); int offset = bytesWritable.getOffset(); int len = bytesWritable.getLength(); RowKeyUtils.seek( bytesWritable, fixedPrefixLength + getBcdEncodedLength(bytes, offset + fixedPrefixLength, len - fixedPrefixLength)); }
private int setStartKey(ImmutableBytesWritable ptr, int offset, int i) { int length = ptr.getOffset() - offset; startKey = copyKey(startKey, length + this.maxKeyLength, ptr.get(), offset, length); startKeyLength = length; // Add separator byte if we're at the end of the buffer, since trailing separator bytes are // stripped if (ptr.getOffset() + ptr.getLength() == offset + length && i - 1 > 0 && !schema.getField(i - 1).getType().isFixedWidth()) { startKey[startKeyLength++] = QueryConstants.SEPARATOR_BYTE; } startKeyLength += setKey(Bound.LOWER, startKey, startKeyLength, i); return length; }
/** * Collect all column values for the same Row. RowKey may be different if indexes are involved, so * it writes a separate record for each unique RowKey * * @param context Current mapper context * @param tableName Table index in tableNames list * @param lkv List of KV values that will be combined in a single ImmutableBytesWritable * @throws IOException * @throws InterruptedException */ private void writeAggregatedRow(Context context, String tableName, List<KeyValue> lkv) throws IOException, InterruptedException { ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); DataOutputStream outputStream = new DataOutputStream(bos); ImmutableBytesWritable outputKey = null; if (!lkv.isEmpty()) { for (KeyValue cell : lkv) { if (outputKey == null || Bytes.compareTo( outputKey.get(), outputKey.getOffset(), outputKey.getLength(), cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()) != 0) { // This a the first RowKey or a different from previous if (outputKey != null) { // It's a different RowKey, so we need to write it ImmutableBytesWritable aggregatedArray = new ImmutableBytesWritable(bos.toByteArray()); outputStream.close(); context.write(new TableRowkeyPair(tableName, outputKey), aggregatedArray); } outputKey = new ImmutableBytesWritable( cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); bos = new ByteArrayOutputStream(1024); outputStream = new DataOutputStream(bos); } /* The order of aggregation: type, index of column, length of value, value itself */ int i = findIndex(cell); if (i == -1) { // That may happen when we load only local indexes. Since KV pairs for both // table and local index are going to the same physical table at that point // we skip those KVs that are not belongs to loca index continue; } outputStream.writeByte(cell.getTypeByte()); WritableUtils.writeVInt(outputStream, i); WritableUtils.writeVInt(outputStream, cell.getValueLength()); outputStream.write(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); } ImmutableBytesWritable aggregatedArray = new ImmutableBytesWritable(bos.toByteArray()); outputStream.close(); context.write(new TableRowkeyPair(tableName, outputKey), aggregatedArray); } }
private void iterate() { if (schema.next(ptr, position++, maxOffset) == null) { nextValue = null; } else { nextValue = ptr.copyBytes(); } }
private static Put resultToPut(ImmutableBytesWritable key, Result result) throws IOException { Put put = new Put(key.get()); for (Cell kv : result.rawCells()) { put.add(kv); } return put; }
// @Override // protected void map(ImmutableBytesWritable key, Text value, Context context) throws // IOException, InterruptedException { // Text combinedKeyValue = new Text(); // //the structure is key###value // combinedKeyValue.set(Bytes.toString(key.get()) + "###" + value.toString()); // context.write(one, combinedKeyValue); // } @Override protected void map(ImmutableBytesWritable key, Result columns, Context context) throws IOException, InterruptedException { Text combinedKeyValue = new Text(); // the structure is key###value String value = null; try { for (KeyValue kv : columns.list()) { byte[] gmmData = kv.getValue(); String gmmString = Bytes.toStringBinary(gmmData); // /* just for checking that gmm is correctly constructed MixtureModel m = null; m = (MixtureModel) ObjectAndByte.byteArrayToObject(Bytes.toBytesBinary(gmmString)); System.out.println("m.size:" + m.size); // */ combinedKeyValue.set(Bytes.toString(key.get()) + "###" + gmmString); context.write(one, combinedKeyValue); // context.write(key, new Text(gmmString)); } } catch (Exception e) { e.printStackTrace(); } }
/** @return The HTableDescriptor to create or alter the table */ protected HTableDescriptor getHTableDescriptor() { HTableDescriptor hTableDescriptor = new HTableDescriptor(tableKey.get()); for (HColumnDescriptor family : this.columnFamilies) { hTableDescriptor.addFamily(family); } return hTableDescriptor; }
public int getPartition(ImmutableBytesWritable key, V2 value, int numPartitions) { byte[] region = null; // Only one region return 0 if (this.startKeys.length == 1) { return 0; } try { // Not sure if this is cached after a split so we could have problems // here if a region splits while mapping region = table.getRegionLocation(key.get()).getRegionInfo().getStartKey(); } catch (IOException e) { LOG.error(e); } for (int i = 0; i < this.startKeys.length; i++) { if (Bytes.compareTo(region, this.startKeys[i]) == 0) { if (i >= numPartitions - 1) { // cover if we have less reduces then regions. return (Integer.toString(i).hashCode() & Integer.MAX_VALUE) % numPartitions; } return i; } } // if above fails to find start key that match we need to return something return 0; }
public PRowImpl(ImmutableBytesWritable key, long ts) { byte[] keyBytes = key.copyBytes(); this.setValues = new Put(keyBytes); this.unsetValues = new Delete(keyBytes); this.ts = ts; this.key = keyBytes; }
@Override public final Object getValue(Tuple tuple, PDataType type, ImmutableBytesWritable ptr) throws SQLException { try { Expression expression = getExpression(); if (!expression.evaluate(tuple, ptr)) { return null; } if (ptr.getLength() == 0) { return null; } return type.toObject( ptr, expression.getDataType(), expression.getSortOrder(), expression.getMaxLength(), expression.getScale()); } catch (RuntimeException e) { // FIXME: Expression.evaluate does not throw SQLException // so this will unwrap throws from that. if (e.getCause() instanceof SQLException) { throw (SQLException) e.getCause(); } throw e; } }
@Override public KeyValue getValue(byte[] family, byte[] qualifier) { if (keyValue == null) { keyValue = KeyValueUtil.newKeyValue( keyPtr.get(), keyPtr.getOffset(), keyPtr.getLength(), VALUE_COLUMN_FAMILY, VALUE_COLUMN_QUALIFIER, timestamp, projectedValue, 0, projectedValue.length); } return keyValue; }
@Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { // Starting from the end of the byte, look for all single bytes at the end of the string // that is below SPACE_UTF8 (space and control characters) or above (control chars). if (!getStringExpression().evaluate(tuple, ptr)) { return false; } if (ptr.getLength() == 0) { ptr.set(ByteUtil.EMPTY_BYTE_ARRAY); return true; } byte[] string = ptr.get(); int offset = ptr.getOffset(); int length = ptr.getLength(); int i = offset + length - 1; for (; i >= offset; i--) { if ((string[i] & SINGLE_BYTE_MASK) == 0 && SPACE_UTF8 < string[i] && string[i] != 0x7f) { break; } } if (i == offset - 1) { ptr.set(ByteUtil.EMPTY_BYTE_ARRAY); return true; } ptr.set(string, offset, i - offset + 1); return true; }
@Override public void map(ImmutableBytesWritable row, Result value, Context context) throws InterruptedException, IOException { KeyValue[] kvs = value.raw(); if (lastRowKey == null) { lastRowKey = row.get(); } else if (Bytes.compareTo(lastRowKey, row.get()) != 0) { writeLine(context, Bytes.toString(lastRowKey)); columnValueMap.clear(); } for (KeyValue kv : kvs) { String qualifier = Bytes.toString(kv.getQualifier()); byte[] val = kv.getValue(); columnValueMap.put(qualifier, val); } }
@Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { Expression arg = getChildren().get(0); if (!arg.evaluate(tuple, ptr)) { return false; } if (ptr.getLength() == 0) { return true; } long dateTime = inputCodec.decodeLong(ptr, arg.getSortOrder()); DateTime jodaDT = new DateTime(dateTime); int day = jodaDT.getDayOfYear(); PDataType returnDataType = getDataType(); byte[] byteValue = new byte[returnDataType.getByteSize()]; returnDataType.getCodec().encodeInt(day, byteValue, 0); ptr.set(byteValue); return true; }