private static Block createFixWidthValueBlock(int positionCount, int mapSize) { BlockBuilder valueBlockBuilder = DOUBLE.createBlockBuilder(new BlockBuilderStatus(), positionCount * mapSize); for (int i = 0; i < positionCount * mapSize; i++) { DOUBLE.writeDouble(valueBlockBuilder, ThreadLocalRandom.current().nextDouble()); } return valueBlockBuilder.build(); }
@Setup public void setup() { Random random = new Random(); RowExpression[] arguments = new RowExpression[1 + inListCount]; switch (type) { case StandardTypes.BIGINT: prestoType = BIGINT; for (int i = 1; i <= inListCount; i++) { arguments[i] = constant((long) random.nextInt(), BIGINT); } break; case StandardTypes.DOUBLE: prestoType = DOUBLE; for (int i = 1; i <= inListCount; i++) { arguments[i] = constant(random.nextDouble(), DOUBLE); } break; case StandardTypes.VARCHAR: prestoType = VARCHAR; for (int i = 1; i <= inListCount; i++) { arguments[i] = constant(Slices.utf8Slice(Long.toString(random.nextLong())), VARCHAR); } break; default: throw new IllegalStateException(); } arguments[0] = field(0, prestoType); RowExpression project = field(0, prestoType); PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(prestoType)); for (int i = 0; i < 10_000; i++) { pageBuilder.declarePosition(); switch (type) { case StandardTypes.BIGINT: BIGINT.writeLong(pageBuilder.getBlockBuilder(0), random.nextInt()); break; case StandardTypes.DOUBLE: DOUBLE.writeDouble(pageBuilder.getBlockBuilder(0), random.nextDouble()); break; case StandardTypes.VARCHAR: VARCHAR.writeSlice( pageBuilder.getBlockBuilder(0), Slices.utf8Slice(Long.toString(random.nextLong()))); break; } } inputPage = pageBuilder.build(); RowExpression filter = call( new Signature(IN, SCALAR, parseTypeSignature(StandardTypes.BOOLEAN)), BOOLEAN, arguments); processor = new ExpressionCompiler(MetadataManager.createTestMetadataManager()) .compilePageProcessor(filter, ImmutableList.of(project)) .get(); }
private void parseColumn(int column) { Type type = types[column]; if (BOOLEAN.equals(type)) { parseBooleanColumn(column); } else if (BIGINT.equals(type)) { parseLongColumn(column); } else if (INTEGER.equals(type)) { parseLongColumn(column); } else if (SMALLINT.equals(type)) { parseLongColumn(column); } else if (TINYINT.equals(type)) { parseLongColumn(column); } else if (DOUBLE.equals(type)) { parseDoubleColumn(column); } else if (isVarcharType(type) || VARBINARY.equals(type)) { parseStringColumn(column); } else if (isStructuralType(hiveTypes[column])) { parseObjectColumn(column); } else if (DATE.equals(type)) { parseLongColumn(column); } else if (TIMESTAMP.equals(type)) { parseLongColumn(column); } else if (type instanceof DecimalType) { parseDecimalColumn(column); } else { throw new UnsupportedOperationException("Unsupported column type: " + type); } }
private void testCorrectnessOfErrorFunction(List<Number> inputList) throws Exception { int inRange = 0; int numberOfRuns = 1000; double sampleRatio = 1 / (double) WEIGHT; double actual = getExpectedValue(inputList); Random rand = new Random(1); for (int i = 0; i < numberOfRuns; i++) { // Compute Sampled Value using sampledList (numberOfRuns times) ImmutableList.Builder<Number> sampledList = ImmutableList.builder(); for (Number x : inputList) { if (rand.nextDouble() < sampleRatio) { sampledList.add(x); } } BlockBuilder builder = getType().createBlockBuilder(new BlockBuilderStatus()); for (Number sample : sampledList.build()) { if (getType() == BIGINT) { BIGINT.writeLong(builder, sample.longValue()); } else if (getType() == DOUBLE) { DOUBLE.writeDouble(builder, sample.doubleValue()); } else { throw new AssertionError("Can only handle longs and doubles"); } } Page page = new Page(builder.build()); page = OperatorAssertion.appendSampleWeight(ImmutableList.of(page), WEIGHT).get(0); Accumulator accumulator = getFunction() .bind( ImmutableList.of(0), Optional.<Integer>absent(), Optional.of(page.getChannelCount() - 1), getConfidence()) .createAccumulator(); accumulator.addInput(page); Block result = accumulator.evaluateFinal(); String approxValue = BlockAssertions.toValues(accumulator.getFinalType(), result).get(0).toString(); double approx = Double.parseDouble(approxValue.split(" ")[0]); double error = Double.parseDouble(approxValue.split(" ")[2]); // Check if actual answer lies within [approxAnswer - error, approxAnswer + error] if (Math.abs(approx - actual) <= error) { inRange++; } } BinomialDistribution binomial = new BinomialDistribution(numberOfRuns, getConfidence()); int lowerBound = binomial.inverseCumulativeProbability(0.01); int upperBound = binomial.inverseCumulativeProbability(0.99); assertTrue( lowerBound < inRange && inRange < upperBound, String.format( "%d out of %d passed. Expected [%d, %d]", inRange, numberOfRuns, lowerBound, upperBound)); }
private static void assertReadFields(RecordCursor cursor, List<ColumnMetadata> schema) { for (int columnIndex = 0; columnIndex < schema.size(); columnIndex++) { ColumnMetadata column = schema.get(columnIndex); if (!cursor.isNull(columnIndex)) { Type type = column.getType(); if (BOOLEAN.equals(type)) { cursor.getBoolean(columnIndex); } else if (BIGINT.equals(type)) { cursor.getLong(columnIndex); } else if (TIMESTAMP.equals(type)) { cursor.getLong(columnIndex); } else if (DOUBLE.equals(type)) { cursor.getDouble(columnIndex); } else if (VARCHAR.equals(type) || VARBINARY.equals(type)) { try { cursor.getSlice(columnIndex); } catch (RuntimeException e) { throw new RuntimeException("column " + column, e); } } else { fail("Unknown primitive type " + columnIndex); } } } }
protected void checkCursor(RecordCursor cursor, List<TestColumn> testColumns, int numRows) throws IOException { for (int row = 0; row < numRows; row++) { assertTrue(cursor.advanceNextPosition()); for (int i = 0, testColumnsSize = testColumns.size(); i < testColumnsSize; i++) { TestColumn testColumn = testColumns.get(i); Object fieldFromCursor; Type type = HiveType.valueOf(testColumn.getObjectInspector().getTypeName()).getType(TYPE_MANAGER); if (cursor.isNull(i)) { fieldFromCursor = null; } else if (BOOLEAN.equals(type)) { fieldFromCursor = cursor.getBoolean(i); } else if (BIGINT.equals(type)) { fieldFromCursor = cursor.getLong(i); } else if (DOUBLE.equals(type)) { fieldFromCursor = cursor.getDouble(i); } else if (VARCHAR.equals(type)) { fieldFromCursor = cursor.getSlice(i); } else if (VARBINARY.equals(type)) { fieldFromCursor = cursor.getSlice(i); } else if (DateType.DATE.equals(type)) { fieldFromCursor = cursor.getLong(i); } else if (TimestampType.TIMESTAMP.equals(type)) { fieldFromCursor = cursor.getLong(i); } else if (isStructuralType(type)) { fieldFromCursor = cursor.getObject(i); } else { throw new RuntimeException("unknown type"); } if (fieldFromCursor == null) { assertEquals( null, testColumn.getExpectedValue(), String.format("Expected null for column %s", testColumn.getName())); } else if (testColumn.getObjectInspector().getTypeName().equals("float") || testColumn.getObjectInspector().getTypeName().equals("double")) { assertEquals((double) fieldFromCursor, (double) testColumn.getExpectedValue(), EPSILON); } else if (testColumn.getObjectInspector().getCategory() == Category.PRIMITIVE) { assertEquals( fieldFromCursor, testColumn.getExpectedValue(), String.format("Wrong value for column %s", testColumn.getName())); } else { Block expected = (Block) testColumn.getExpectedValue(); Block actual = (Block) fieldFromCursor; assertBlockEquals( actual, expected, String.format("Wrong value for column %s", testColumn.getName())); } } } }
@Override public Block getSequenceBlock(int start, int length) { BlockBuilder blockBuilder = getType().createBlockBuilder(new BlockBuilderStatus()); for (int i = start; i < start + length; i++) { if (getType() == BIGINT) { BIGINT.writeLong(blockBuilder, (long) i); } else { DOUBLE.writeDouble(blockBuilder, (double) i); } } return blockBuilder.build(); }
public static ParameterType getById(final Long id) { if (STRING.getId().equals(id)) { return STRING; } else if (LONG.getId().equals(id)) { return LONG; } else if (DATE.getId().equals(id)) { return DATE; } else if (DOUBLE.getId().equals(id)) { return DOUBLE; } else { throw new SpringBatchLightminApplicationException("Unknown ParameterType for id: " + id); } }
private static void insertRows( ConnectorTableMetadata tableMetadata, Handle handle, RecordSet data) { List<ColumnMetadata> columns = ImmutableList.copyOf( Iterables.filter( tableMetadata.getColumns(), new Predicate<ColumnMetadata>() { @Override public boolean apply(ColumnMetadata columnMetadata) { return !columnMetadata.isHidden(); } })); String vars = Joiner.on(',').join(nCopies(columns.size(), "?")); String sql = format("INSERT INTO %s VALUES (%s)", tableMetadata.getTable().getTableName(), vars); RecordCursor cursor = data.cursor(); while (true) { // insert 1000 rows at a time PreparedBatch batch = handle.prepareBatch(sql); for (int row = 0; row < 1000; row++) { if (!cursor.advanceNextPosition()) { batch.execute(); return; } PreparedBatchPart part = batch.add(); for (int column = 0; column < columns.size(); column++) { Type type = columns.get(column).getType(); if (BOOLEAN.equals(type)) { part.bind(column, cursor.getBoolean(column)); } else if (BIGINT.equals(type)) { part.bind(column, cursor.getLong(column)); } else if (DOUBLE.equals(type)) { part.bind(column, cursor.getDouble(column)); } else if (VARCHAR.equals(type)) { part.bind(column, cursor.getSlice(column).toStringUtf8()); } else if (DATE.equals(type)) { long millisUtc = TimeUnit.DAYS.toMillis(cursor.getLong(column)); // H2 expects dates in to be millis at midnight in the JVM timezone long localMillis = DateTimeZone.UTC.getMillisKeepLocal(DateTimeZone.getDefault(), millisUtc); part.bind(column, new Date(localMillis)); } else { throw new IllegalArgumentException("Unsupported type " + type); } } } batch.execute(); } }
public static PropertyDataTypes getTypeByTypeNum(int typeNum) { if (STRING.getTypeNum() == typeNum) return STRING; else if (BINARY.getTypeNum() == typeNum) return BINARY; else if (LONG.getTypeNum() == typeNum) return LONG; else if (DOUBLE.getTypeNum() == typeNum) return DOUBLE; else if (DATE.getTypeNum() == typeNum) return DATE; else if (BOOLEAN.getTypeNum() == typeNum) return BOOLEAN; else if (NAME.getTypeNum() == typeNum) return NAME; else if (PATH.getTypeNum() == typeNum) return PATH; else if (REFERENCE.getTypeNum() == typeNum) return REFERENCE; else if (WEAKREFERENCE.getTypeNum() == typeNum) return WEAKREFERENCE; else if (URI.getTypeNum() == typeNum) return URI; else if (DECIMAL.getTypeNum() == typeNum) return DECIMAL; else return UNDEFINED; }
private void parseColumn(int column) { Type type = types[column]; if (BOOLEAN.equals(type)) { parseBooleanColumn(column); } else if (BIGINT.equals(type)) { parseLongColumn(column); } else if (DOUBLE.equals(type)) { parseDoubleColumn(column); } else if (VARCHAR.equals(type) || VARBINARY.equals(type)) { parseStringColumn(column); } else if (TIMESTAMP.equals(type)) { parseLongColumn(column); } else { throw new UnsupportedOperationException("Unsupported column type: " + type); } }
@OutputFunction("array<double>") public static void output(DigestAndPercentileArrayState state, BlockBuilder out) { QuantileDigest digest = state.getDigest(); List<Double> percentiles = state.getPercentiles(); if (percentiles == null || digest == null) { out.appendNull(); return; } BlockBuilder blockBuilder = out.beginBlockEntry(); for (int i = 0; i < percentiles.size(); i++) { Double percentile = percentiles.get(i); DOUBLE.writeDouble(blockBuilder, sortableLongToDouble(digest.getQuantile(percentile))); } out.closeEntry(); }
@Override public void addInput(Page page) { Block block = page.getBlock(labelChannel); for (int position = 0; position < block.getPositionCount(); position++) { if (labelIsLong) { labels.add((double) BIGINT.getLong(block, position)); } else { labels.add(DOUBLE.getDouble(block, position)); } } block = page.getBlock(featuresChannel); for (int position = 0; position < block.getPositionCount(); position++) { FeatureVector featureVector = ModelUtils.jsonToFeatures(VARCHAR.getSlice(block, position)); rowsSize += featureVector.getEstimatedSize(); rows.add(featureVector); } if (params == null) { block = page.getBlock(paramsChannel); params = LibSvmUtils.parseParameters(VARCHAR.getSlice(block, 0).toStringUtf8()); } }
public static NullableValue parsePartitionValue( String partitionName, String value, Type type, DateTimeZone timeZone) { boolean isNull = HIVE_DEFAULT_DYNAMIC_PARTITION.equals(value); if (type instanceof DecimalType) { DecimalType decimalType = (DecimalType) type; if (isNull) { return NullableValue.asNull(decimalType); } if (decimalType.isShort()) { if (value.isEmpty()) { return NullableValue.of(decimalType, 0L); } return NullableValue.of( decimalType, shortDecimalPartitionKey(value, decimalType, partitionName)); } else { if (value.isEmpty()) { return NullableValue.of(decimalType, Decimals.encodeUnscaledValue(BigInteger.ZERO)); } return NullableValue.of( decimalType, longDecimalPartitionKey(value, decimalType, partitionName)); } } if (BOOLEAN.equals(type)) { if (isNull) { return NullableValue.asNull(BOOLEAN); } if (value.isEmpty()) { return NullableValue.of(BOOLEAN, false); } return NullableValue.of(BOOLEAN, booleanPartitionKey(value, partitionName)); } if (TINYINT.equals(type)) { if (isNull) { return NullableValue.asNull(TINYINT); } if (value.isEmpty()) { return NullableValue.of(TINYINT, 0L); } return NullableValue.of(TINYINT, tinyintPartitionKey(value, partitionName)); } if (SMALLINT.equals(type)) { if (isNull) { return NullableValue.asNull(SMALLINT); } if (value.isEmpty()) { return NullableValue.of(SMALLINT, 0L); } return NullableValue.of(SMALLINT, smallintPartitionKey(value, partitionName)); } if (INTEGER.equals(type)) { if (isNull) { return NullableValue.asNull(INTEGER); } if (value.isEmpty()) { return NullableValue.of(INTEGER, 0L); } return NullableValue.of(INTEGER, integerPartitionKey(value, partitionName)); } if (BIGINT.equals(type)) { if (isNull) { return NullableValue.asNull(BIGINT); } if (value.isEmpty()) { return NullableValue.of(BIGINT, 0L); } return NullableValue.of(BIGINT, bigintPartitionKey(value, partitionName)); } if (DATE.equals(type)) { if (isNull) { return NullableValue.asNull(DATE); } return NullableValue.of(DATE, datePartitionKey(value, partitionName)); } if (TIMESTAMP.equals(type)) { if (isNull) { return NullableValue.asNull(TIMESTAMP); } return NullableValue.of(TIMESTAMP, timestampPartitionKey(value, timeZone, partitionName)); } if (REAL.equals(type)) { if (isNull) { return NullableValue.asNull(REAL); } if (value.isEmpty()) { return NullableValue.of(REAL, (long) floatToRawIntBits(0.0f)); } return NullableValue.of(REAL, floatPartitionKey(value, partitionName)); } if (DOUBLE.equals(type)) { if (isNull) { return NullableValue.asNull(DOUBLE); } if (value.isEmpty()) { return NullableValue.of(DOUBLE, 0.0); } return NullableValue.of(DOUBLE, doublePartitionKey(value, partitionName)); } if (type instanceof VarcharType) { if (isNull) { return NullableValue.asNull(type); } return NullableValue.of(type, varcharPartitionKey(value, partitionName, type)); } if (isCharType(type)) { if (isNull) { return NullableValue.asNull(type); } return NullableValue.of(type, charPartitionKey(value, partitionName, type)); } throw new PrestoException( NOT_SUPPORTED, format("Unsupported Type [%s] for partition: %s", type, partitionName)); }
public GenericHiveRecordCursor( RecordReader<K, V> recordReader, long totalBytes, Properties splitSchema, List<HivePartitionKey> partitionKeys, List<HiveColumnHandle> columns, DateTimeZone hiveStorageTimeZone, DateTimeZone sessionTimeZone) { checkNotNull(recordReader, "recordReader is null"); checkArgument(totalBytes >= 0, "totalBytes is negative"); checkNotNull(splitSchema, "splitSchema is null"); checkNotNull(partitionKeys, "partitionKeys is null"); checkNotNull(columns, "columns is null"); checkArgument(!columns.isEmpty(), "columns is empty"); checkNotNull(hiveStorageTimeZone, "hiveStorageTimeZone is null"); checkNotNull(sessionTimeZone, "sessionTimeZone is null"); this.recordReader = recordReader; this.totalBytes = totalBytes; this.key = recordReader.createKey(); this.value = recordReader.createValue(); this.hiveStorageTimeZone = hiveStorageTimeZone; this.sessionTimeZone = sessionTimeZone; this.deserializer = getDeserializer(splitSchema); this.rowInspector = getTableObjectInspector(deserializer); int size = columns.size(); String[] names = new String[size]; this.types = new Type[size]; this.hiveTypes = new HiveType[size]; this.structFields = new StructField[size]; this.fieldInspectors = new ObjectInspector[size]; this.isPartitionColumn = new boolean[size]; this.loaded = new boolean[size]; this.booleans = new boolean[size]; this.longs = new long[size]; this.doubles = new double[size]; this.slices = new Slice[size]; this.nulls = new boolean[size]; // initialize data columns for (int i = 0; i < columns.size(); i++) { HiveColumnHandle column = columns.get(i); names[i] = column.getName(); types[i] = column.getType(); hiveTypes[i] = column.getHiveType(); if (!column.isPartitionKey()) { StructField field = rowInspector.getStructFieldRef(column.getName()); structFields[i] = field; fieldInspectors[i] = field.getFieldObjectInspector(); } isPartitionColumn[i] = column.isPartitionKey(); } // parse requested partition columns Map<String, HivePartitionKey> partitionKeysByName = uniqueIndex(partitionKeys, HivePartitionKey.nameGetter()); for (int columnIndex = 0; columnIndex < columns.size(); columnIndex++) { HiveColumnHandle column = columns.get(columnIndex); if (column.isPartitionKey()) { HivePartitionKey partitionKey = partitionKeysByName.get(column.getName()); checkArgument(partitionKey != null, "Unknown partition key %s", column.getName()); byte[] bytes = partitionKey.getValue().getBytes(Charsets.UTF_8); Type type = types[columnIndex]; if (BOOLEAN.equals(type)) { if (isTrue(bytes, 0, bytes.length)) { booleans[columnIndex] = true; } else if (isFalse(bytes, 0, bytes.length)) { booleans[columnIndex] = false; } else { String valueString = new String(bytes, Charsets.UTF_8); throw new IllegalArgumentException( String.format( "Invalid partition value '%s' for BOOLEAN partition key %s", valueString, names[columnIndex])); } } else if (BIGINT.equals(type)) { if (bytes.length == 0) { throw new IllegalArgumentException( String.format( "Invalid partition value '' for BIGINT partition key %s", names[columnIndex])); } longs[columnIndex] = parseLong(bytes, 0, bytes.length); } else if (DOUBLE.equals(type)) { if (bytes.length == 0) { throw new IllegalArgumentException( String.format( "Invalid partition value '' for DOUBLE partition key %s", names[columnIndex])); } doubles[columnIndex] = parseDouble(bytes, 0, bytes.length); } else if (VARCHAR.equals(type)) { slices[columnIndex] = Slices.wrappedBuffer(Arrays.copyOf(bytes, bytes.length)); } else { throw new UnsupportedOperationException("Unsupported column type: " + type); } } } }
@Override public void addDouble(double value) { DOUBLE.writeDouble(builder, value); wroteValue = true; }
@Test public void testWriter() throws Exception { List<Long> columnIds = ImmutableList.of(1L, 2L, 4L, 6L, 7L, 8L, 9L, 10L); ArrayType arrayType = new ArrayType(BIGINT); ArrayType arrayOfArrayType = new ArrayType(arrayType); MapType mapType = new MapType(createVarcharType(10), BOOLEAN); List<Type> columnTypes = ImmutableList.of( BIGINT, createVarcharType(10), VARBINARY, DOUBLE, BOOLEAN, arrayType, mapType, arrayOfArrayType); File file = new File(directory, System.nanoTime() + ".orc"); byte[] bytes1 = octets(0x00, 0xFE, 0xFF); byte[] bytes3 = octets(0x01, 0x02, 0x19, 0x80); RowPagesBuilder rowPagesBuilder = RowPagesBuilder.rowPagesBuilder(columnTypes) .row( 123L, "hello", wrappedBuffer(bytes1), 123.456, true, arrayBlockOf(BIGINT, 1, 2), mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5))) .row( null, "world", null, Double.POSITIVE_INFINITY, null, arrayBlockOf(BIGINT, 3, null), mapBlockOf(createVarcharType(5), BOOLEAN, "k2", null), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 6, 7))) .row( 456L, "bye \u2603", wrappedBuffer(bytes3), Double.NaN, false, arrayBlockOf(BIGINT), mapBlockOf(createVarcharType(5), BOOLEAN, "k3", false), arrayBlockOf(arrayType, arrayBlockOf(BIGINT))); try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(new EmptyClassLoader()); OrcFileWriter writer = new OrcFileWriter(columnIds, columnTypes, file)) { writer.appendPages(rowPagesBuilder.build()); } try (OrcDataSource dataSource = fileOrcDataSource(file)) { OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes); assertEquals(reader.getReaderRowCount(), 3); assertEquals(reader.getReaderPosition(), 0); assertEquals(reader.getFileRowCount(), reader.getReaderRowCount()); assertEquals(reader.getFilePosition(), reader.getFilePosition()); assertEquals(reader.nextBatch(), 3); assertEquals(reader.getReaderPosition(), 0); assertEquals(reader.getFilePosition(), reader.getFilePosition()); Block column0 = reader.readBlock(BIGINT, 0); assertEquals(column0.isNull(0), false); assertEquals(column0.isNull(1), true); assertEquals(column0.isNull(2), false); assertEquals(BIGINT.getLong(column0, 0), 123L); assertEquals(BIGINT.getLong(column0, 2), 456L); Block column1 = reader.readBlock(createVarcharType(10), 1); assertEquals(createVarcharType(10).getSlice(column1, 0), utf8Slice("hello")); assertEquals(createVarcharType(10).getSlice(column1, 1), utf8Slice("world")); assertEquals(createVarcharType(10).getSlice(column1, 2), utf8Slice("bye \u2603")); Block column2 = reader.readBlock(VARBINARY, 2); assertEquals(VARBINARY.getSlice(column2, 0), wrappedBuffer(bytes1)); assertEquals(column2.isNull(1), true); assertEquals(VARBINARY.getSlice(column2, 2), wrappedBuffer(bytes3)); Block column3 = reader.readBlock(DOUBLE, 3); assertEquals(column3.isNull(0), false); assertEquals(column3.isNull(1), false); assertEquals(column3.isNull(2), false); assertEquals(DOUBLE.getDouble(column3, 0), 123.456); assertEquals(DOUBLE.getDouble(column3, 1), Double.POSITIVE_INFINITY); assertEquals(DOUBLE.getDouble(column3, 2), Double.NaN); Block column4 = reader.readBlock(BOOLEAN, 4); assertEquals(column4.isNull(0), false); assertEquals(column4.isNull(1), true); assertEquals(column4.isNull(2), false); assertEquals(BOOLEAN.getBoolean(column4, 0), true); assertEquals(BOOLEAN.getBoolean(column4, 2), false); Block column5 = reader.readBlock(arrayType, 5); assertEquals(column5.getPositionCount(), 3); assertTrue( arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 0), arrayBlockOf(BIGINT, 1, 2))); assertTrue( arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 1), arrayBlockOf(BIGINT, 3, null))); assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 2), arrayBlockOf(BIGINT))); Block column6 = reader.readBlock(mapType, 6); assertEquals(column6.getPositionCount(), 3); assertTrue( mapBlocksEqual( createVarcharType(5), BOOLEAN, arrayType.getObject(column6, 0), mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true))); assertTrue( mapBlocksEqual( createVarcharType(5), BOOLEAN, arrayType.getObject(column6, 1), mapBlockOf(createVarcharType(5), BOOLEAN, "k2", null))); assertTrue( mapBlocksEqual( createVarcharType(5), BOOLEAN, arrayType.getObject(column6, 2), mapBlockOf(createVarcharType(5), BOOLEAN, "k3", false))); Block column7 = reader.readBlock(arrayOfArrayType, 7); assertEquals(column7.getPositionCount(), 3); assertTrue( arrayBlocksEqual( arrayType, arrayOfArrayType.getObject(column7, 0), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5)))); assertTrue( arrayBlocksEqual( arrayType, arrayOfArrayType.getObject(column7, 1), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 6, 7)))); assertTrue( arrayBlocksEqual( arrayType, arrayOfArrayType.getObject(column7, 2), arrayBlockOf(arrayType, arrayBlockOf(BIGINT)))); assertEquals(reader.nextBatch(), -1); assertEquals(reader.getReaderPosition(), 3); assertEquals(reader.getFilePosition(), reader.getFilePosition()); OrcFileMetadata orcFileMetadata = METADATA_CODEC.fromJson(reader.getUserMetadata().get(OrcFileMetadata.KEY).getBytes()); assertEquals( orcFileMetadata, new OrcFileMetadata( ImmutableMap.<Long, TypeSignature>builder() .put(1L, BIGINT.getTypeSignature()) .put(2L, createVarcharType(10).getTypeSignature()) .put(4L, VARBINARY.getTypeSignature()) .put(6L, DOUBLE.getTypeSignature()) .put(7L, BOOLEAN.getTypeSignature()) .put(8L, arrayType.getTypeSignature()) .put(9L, mapType.getTypeSignature()) .put(10L, arrayOfArrayType.getTypeSignature()) .build())); } File crcFile = new File(file.getParentFile(), "." + file.getName() + ".crc"); assertFalse(crcFile.exists()); }
@Override public void addFloat(float value) { DOUBLE.writeDouble(builder, value); wroteValue = true; }
public GenericHiveRecordCursor( RecordReader<K, V> recordReader, long totalBytes, Properties splitSchema, List<HivePartitionKey> partitionKeys, List<HiveColumnHandle> columns, DateTimeZone hiveStorageTimeZone, TypeManager typeManager) { requireNonNull(recordReader, "recordReader is null"); checkArgument(totalBytes >= 0, "totalBytes is negative"); requireNonNull(splitSchema, "splitSchema is null"); requireNonNull(partitionKeys, "partitionKeys is null"); requireNonNull(columns, "columns is null"); requireNonNull(hiveStorageTimeZone, "hiveStorageTimeZone is null"); this.recordReader = recordReader; this.totalBytes = totalBytes; this.key = recordReader.createKey(); this.value = recordReader.createValue(); this.hiveStorageTimeZone = hiveStorageTimeZone; this.deserializer = getDeserializer(splitSchema); this.rowInspector = getTableObjectInspector(deserializer); int size = columns.size(); String[] names = new String[size]; this.types = new Type[size]; this.hiveTypes = new HiveType[size]; this.structFields = new StructField[size]; this.fieldInspectors = new ObjectInspector[size]; this.isPartitionColumn = new boolean[size]; this.loaded = new boolean[size]; this.booleans = new boolean[size]; this.longs = new long[size]; this.doubles = new double[size]; this.slices = new Slice[size]; this.objects = new Object[size]; this.nulls = new boolean[size]; // initialize data columns for (int i = 0; i < columns.size(); i++) { HiveColumnHandle column = columns.get(i); names[i] = column.getName(); types[i] = typeManager.getType(column.getTypeSignature()); hiveTypes[i] = column.getHiveType(); if (!column.isPartitionKey()) { StructField field = rowInspector.getStructFieldRef(column.getName()); structFields[i] = field; fieldInspectors[i] = field.getFieldObjectInspector(); } isPartitionColumn[i] = column.isPartitionKey(); } // parse requested partition columns Map<String, HivePartitionKey> partitionKeysByName = uniqueIndex(partitionKeys, HivePartitionKey::getName); for (int columnIndex = 0; columnIndex < columns.size(); columnIndex++) { HiveColumnHandle column = columns.get(columnIndex); if (column.isPartitionKey()) { HivePartitionKey partitionKey = partitionKeysByName.get(column.getName()); checkArgument(partitionKey != null, "Unknown partition key %s", column.getName()); byte[] bytes = partitionKey.getValue().getBytes(UTF_8); String name = names[columnIndex]; Type type = types[columnIndex]; if (HiveUtil.isHiveNull(bytes)) { nulls[columnIndex] = true; } else if (BOOLEAN.equals(type)) { booleans[columnIndex] = booleanPartitionKey(partitionKey.getValue(), name); } else if (BIGINT.equals(type)) { longs[columnIndex] = bigintPartitionKey(partitionKey.getValue(), name); } else if (INTEGER.equals(type)) { longs[columnIndex] = integerPartitionKey(partitionKey.getValue(), name); } else if (SMALLINT.equals(type)) { longs[columnIndex] = smallintPartitionKey(partitionKey.getValue(), name); } else if (TINYINT.equals(type)) { longs[columnIndex] = tinyintPartitionKey(partitionKey.getValue(), name); } else if (DOUBLE.equals(type)) { doubles[columnIndex] = doublePartitionKey(partitionKey.getValue(), name); } else if (isVarcharType(type)) { slices[columnIndex] = varcharPartitionKey(partitionKey.getValue(), name, type); } else if (DATE.equals(type)) { longs[columnIndex] = datePartitionKey(partitionKey.getValue(), name); } else if (TIMESTAMP.equals(type)) { longs[columnIndex] = timestampPartitionKey(partitionKey.getValue(), hiveStorageTimeZone, name); } else if (isShortDecimal(type)) { longs[columnIndex] = shortDecimalPartitionKey(partitionKey.getValue(), (DecimalType) type, name); } else if (isLongDecimal(type)) { slices[columnIndex] = longDecimalPartitionKey(partitionKey.getValue(), (DecimalType) type, name); } else { throw new PrestoException( NOT_SUPPORTED, format( "Unsupported column type %s for partition key: %s", type.getDisplayName(), name)); } } } }
public class TestApproximatePercentileAggregation { private static final Block EMPTY_DOUBLE_BLOCK = DOUBLE.createBlockBuilder(new BlockBuilderStatus(), 0).build(); private static final Block EMPTY_LONG_BLOCK = BIGINT.createBlockBuilder(new BlockBuilderStatus(), 0).build(); @Test public void testLongPartialStep() throws Exception { // regular approx_percentile assertAggregation( LONG_APPROXIMATE_PERCENTILE_AGGREGATION, 1.0, null, createLongsBlock(null, null), createRLEBlock(0.5, 2)); assertAggregation( LONG_APPROXIMATE_PERCENTILE_AGGREGATION, 1.0, 1L, createLongsBlock(null, 1L), createRLEBlock(0.5, 2)); assertAggregation( LONG_APPROXIMATE_PERCENTILE_AGGREGATION, 1.0, 2L, createLongsBlock(null, 1L, 2L, 3L), createRLEBlock(0.5, 4)); assertAggregation( LONG_APPROXIMATE_PERCENTILE_AGGREGATION, 1.0, 2L, createLongsBlock(1L, 2L, 3L), createRLEBlock(0.5, 3)); assertAggregation( LONG_APPROXIMATE_PERCENTILE_AGGREGATION, 1.0, 3L, createLongsBlock( 1L, null, 2L, 2L, null, 2L, 2L, null, 2L, 2L, null, 3L, 3L, null, 3L, null, 3L, 4L, 5L, 6L, 7L), createRLEBlock(0.5, 21)); // weighted approx_percentile assertAggregation( LONG_APPROXIMATE_PERCENTILE_WEIGHTED_AGGREGATION, 1.0, null, createLongsBlock(null, null), createLongsBlock(1L, 1L), createRLEBlock(0.5, 2)); assertAggregation( LONG_APPROXIMATE_PERCENTILE_WEIGHTED_AGGREGATION, 1.0, 1L, createLongsBlock(null, 1L), createLongsBlock(1L, 1L), createRLEBlock(0.5, 2)); assertAggregation( LONG_APPROXIMATE_PERCENTILE_WEIGHTED_AGGREGATION, 1.0, 2L, createLongsBlock(null, 1L, 2L, 3L), createLongsBlock(1L, 1L, 1L, 1L), createRLEBlock(0.5, 4)); assertAggregation( LONG_APPROXIMATE_PERCENTILE_WEIGHTED_AGGREGATION, 1.0, 2L, createLongsBlock(1L, 2L, 3L), createLongsBlock(1L, 1L, 1L), createRLEBlock(0.5, 3)); assertAggregation( LONG_APPROXIMATE_PERCENTILE_WEIGHTED_AGGREGATION, 1.0, 3L, createLongsBlock( 1L, null, 2L, null, 2L, null, 2L, null, 3L, null, 3L, null, 3L, 4L, 5L, 6L, 7L), createLongsBlock(1L, 1L, 2L, 1L, 2L, 1L, 2L, 1L, 2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L), createRLEBlock(0.5, 17)); } @Test public void testDoublePartialStep() throws Exception { // regular approx_percentile assertAggregation( DOUBLE_APPROXIMATE_PERCENTILE_AGGREGATION, 1.0, null, createDoublesBlock(null, null), createRLEBlock(0.5, 2)); assertAggregation( DOUBLE_APPROXIMATE_PERCENTILE_AGGREGATION, 1.0, 1.0, createDoublesBlock(null, 1.0), createRLEBlock(0.5, 2)); assertAggregation( DOUBLE_APPROXIMATE_PERCENTILE_AGGREGATION, 1.0, 2.0, createDoublesBlock(null, 1.0, 2.0, 3.0), createRLEBlock(0.5, 4)); assertAggregation( DOUBLE_APPROXIMATE_PERCENTILE_AGGREGATION, 1.0, 2.0, createDoublesBlock(1.0, 2.0, 3.0), createRLEBlock(0.5, 3)); assertAggregation( DOUBLE_APPROXIMATE_PERCENTILE_AGGREGATION, 1.0, 3.0, createDoublesBlock( 1.0, null, 2.0, 2.0, null, 2.0, 2.0, null, 2.0, 2.0, null, 3.0, 3.0, null, 3.0, null, 3.0, 4.0, 5.0, 6.0, 7.0), createRLEBlock(0.5, 21)); // weighted approx_percentile assertAggregation( DOUBLE_APPROXIMATE_PERCENTILE_WEIGHTED_AGGREGATION, 1.0, null, createDoublesBlock(null, null), createLongsBlock(1L, 1L), createRLEBlock(0.5, 2)); assertAggregation( DOUBLE_APPROXIMATE_PERCENTILE_WEIGHTED_AGGREGATION, 1.0, 1.0, createDoublesBlock(null, 1.0), createLongsBlock(1L, 1L), createRLEBlock(0.5, 2)); assertAggregation( DOUBLE_APPROXIMATE_PERCENTILE_WEIGHTED_AGGREGATION, 1.0, 2.0, createDoublesBlock(null, 1.0, 2.0, 3.0), createLongsBlock(1L, 1L, 1L, 1L), createRLEBlock(0.5, 4)); assertAggregation( DOUBLE_APPROXIMATE_PERCENTILE_WEIGHTED_AGGREGATION, 1.0, 2.0, createDoublesBlock(1.0, 2.0, 3.0), createLongsBlock(1L, 1L, 1L), createRLEBlock(0.5, 3)); assertAggregation( DOUBLE_APPROXIMATE_PERCENTILE_WEIGHTED_AGGREGATION, 1.0, 3.0, createDoublesBlock( 1.0, null, 2.0, null, 2.0, null, 2.0, null, 3.0, null, 3.0, null, 3.0, 4.0, 5.0, 6.0, 7.0), createLongsBlock(1L, 1L, 2L, 1L, 2L, 1L, 2L, 1L, 2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L), createRLEBlock(0.5, 17)); } }