public static parquet.schema.Type getParquetType( HiveColumnHandle column, MessageType messageType, boolean useParquetColumnNames) { if (useParquetColumnNames) { return getParquetTypeByName(column.getName(), messageType); } if (column.getHiveColumnIndex() < messageType.getFieldCount()) { return messageType.getType(column.getHiveColumnIndex()); } return null; }
public ParquetPageSource( ParquetReader parquetReader, ParquetDataSource dataSource, MessageType fileSchema, MessageType requestedSchema, long totalBytes, Properties splitSchema, List<HiveColumnHandle> columns, TupleDomain<HiveColumnHandle> effectivePredicate, TypeManager typeManager, boolean useParquetColumnNames) { checkArgument(totalBytes >= 0, "totalBytes is negative"); requireNonNull(splitSchema, "splitSchema is null"); requireNonNull(columns, "columns is null"); requireNonNull(effectivePredicate, "effectivePredicate is null"); this.parquetReader = parquetReader; this.dataSource = dataSource; this.requestedSchema = requestedSchema; this.totalBytes = totalBytes; int size = columns.size(); this.constantBlocks = new Block[size]; this.hiveColumnIndexes = new int[size]; ImmutableList.Builder<String> namesBuilder = ImmutableList.builder(); ImmutableList.Builder<Type> typesBuilder = ImmutableList.builder(); for (int columnIndex = 0; columnIndex < size; columnIndex++) { HiveColumnHandle column = columns.get(columnIndex); checkState(column.getColumnType() == REGULAR, "column type must be regular"); String name = column.getName(); Type type = typeManager.getType(column.getTypeSignature()); namesBuilder.add(name); typesBuilder.add(type); hiveColumnIndexes[columnIndex] = column.getHiveColumnIndex(); if (getParquetType(column, fileSchema, useParquetColumnNames) == null) { BlockBuilder blockBuilder = type.createBlockBuilder(new BlockBuilderStatus(), MAX_VECTOR_LENGTH); for (int i = 0; i < MAX_VECTOR_LENGTH; i++) { blockBuilder.appendNull(); } constantBlocks[columnIndex] = blockBuilder.build(); } } types = typesBuilder.build(); columnNames = namesBuilder.build(); }
@Override public Map<String, ColumnHandle> getColumnHandles(ConnectorTableHandle tableHandle) { SchemaTableName tableName = schemaTableName(tableHandle); try { Table table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); ImmutableMap.Builder<String, ColumnHandle> columnHandles = ImmutableMap.builder(); for (HiveColumnHandle columnHandle : hiveColumnHandles(typeManager, connectorId, table, false)) { columnHandles.put(columnHandle.getName(), columnHandle); } return columnHandles.build(); } catch (NoSuchObjectException e) { throw new TableNotFoundException(tableName); } }
@Override public ColumnHandle getSampleWeightColumnHandle(ConnectorTableHandle tableHandle) { SchemaTableName tableName = schemaTableName(tableHandle); try { Table table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()); for (HiveColumnHandle columnHandle : hiveColumnHandles(typeManager, connectorId, table, true)) { if (columnHandle.getName().equals(SAMPLE_WEIGHT_COLUMN_NAME)) { return columnHandle; } } return null; } catch (NoSuchObjectException e) { throw new TableNotFoundException(tableName); } }
public static OrcPageSource createOrcPageSource( MetadataReader metadataReader, Configuration configuration, Path path, long start, long length, List<HiveColumnHandle> columns, List<HivePartitionKey> partitionKeys, TupleDomain<HiveColumnHandle> effectivePredicate, DateTimeZone hiveStorageTimeZone, TypeManager typeManager, DataSize maxMergeDistance, DataSize maxBufferSize, DataSize streamBufferSize) { OrcDataSource orcDataSource; try { FileSystem fileSystem = path.getFileSystem(configuration); long size = fileSystem.getFileStatus(path).getLen(); FSDataInputStream inputStream = fileSystem.open(path); orcDataSource = new HdfsOrcDataSource( path.toString(), size, maxMergeDistance, maxBufferSize, streamBufferSize, inputStream); } catch (Exception e) { if (nullToEmpty(e.getMessage()).trim().equals("Filesystem closed") || e instanceof FileNotFoundException) { throw new PrestoException(HIVE_CANNOT_OPEN_SPLIT, e); } throw new PrestoException(HIVE_CANNOT_OPEN_SPLIT, splitError(e, path, start, length), e); } ImmutableSet.Builder<Integer> includedColumns = ImmutableSet.builder(); ImmutableList.Builder<ColumnReference<HiveColumnHandle>> columnReferences = ImmutableList.builder(); for (HiveColumnHandle column : columns) { if (!column.isPartitionKey()) { includedColumns.add(column.getHiveColumnIndex()); Type type = typeManager.getType(column.getTypeSignature()); columnReferences.add(new ColumnReference<>(column, column.getHiveColumnIndex(), type)); } } OrcPredicate predicate = new TupleDomainOrcPredicate<>(effectivePredicate, columnReferences.build()); try { OrcReader reader = new OrcReader(orcDataSource, metadataReader); OrcRecordReader recordReader = reader.createRecordReader( includedColumns.build(), predicate, start, length, hiveStorageTimeZone); return new OrcPageSource( recordReader, orcDataSource, partitionKeys, columns, hiveStorageTimeZone, typeManager); } catch (Exception e) { try { orcDataSource.close(); } catch (IOException ignored) { } if (e instanceof PrestoException) { throw (PrestoException) e; } String message = splitError(e, path, start, length); if (e.getClass().getSimpleName().equals("BlockMissingException")) { throw new PrestoException(HIVE_MISSING_DATA, message, e); } throw new PrestoException(HIVE_CANNOT_OPEN_SPLIT, message, e); } }