예제 #1
0
 @Override
 public void copyAsField(String name, MapWriter writer) {
   if (writer.ok()) {
     SingleMapWriter impl = (SingleMapWriter) writer.map(name);
     impl.inform(impl.container.copyFromSafe(idx(), impl.idx(), vector));
   }
 }
예제 #2
0
 @Override
 public void copyAsValue(MapWriter writer) {
   if (writer.ok()) {
     SingleMapWriter impl = (SingleMapWriter) writer;
     impl.inform(impl.container.copyFromSafe(idx(), impl.idx(), vector));
   }
 }
  // This function assumes that the fields in the schema parameter are in the same order as the
  // fields in the columns parameter. The
  // columns parameter may have fields that are not present in the schema, though.
  public DrillParquetGroupConverter(
      OutputMutator mutator,
      MapWriter mapWriter,
      GroupType schema,
      Collection<SchemaPath> columns,
      OptionManager options) {
    this.mapWriter = mapWriter;
    this.mutator = mutator;
    converters = Lists.newArrayList();
    this.options = options;

    Iterator<SchemaPath> colIterator = columns.iterator();

    for (Type type : schema.getFields()) {
      Repetition rep = type.getRepetition();
      boolean isPrimitive = type.isPrimitive();

      // Match the name of the field in the schema definition to the name of the field in the query.
      String name = null;
      SchemaPath col = null;
      PathSegment colPath = null;
      PathSegment colNextChild = null;
      while (colIterator.hasNext()) {
        col = colIterator.next();
        colPath = col.getRootSegment();
        colNextChild = colPath.getChild();

        if (colPath != null
            && colPath.isNamed()
            && (!colPath.getNameSegment().getPath().equals("*"))) {
          name = colPath.getNameSegment().getPath();
          // We may have a field that does not exist in the schema
          if (!name.equalsIgnoreCase(type.getName())) {
            continue;
          }
        }
        break;
      }
      if (name == null) {
        name = type.getName();
      }

      if (!isPrimitive) {
        Collection<SchemaPath> c = new ArrayList<SchemaPath>();

        while (colNextChild != null) {
          if (colNextChild.isNamed()) {
            break;
          }
          colNextChild = colNextChild.getChild();
        }

        if (colNextChild != null) {
          SchemaPath s = new SchemaPath(colNextChild.getNameSegment());
          c.add(s);
        }
        if (rep != Repetition.REPEATED) {
          DrillParquetGroupConverter converter =
              new DrillParquetGroupConverter(
                  mutator, mapWriter.map(name), type.asGroupType(), c, options);
          converters.add(converter);
        } else {
          DrillParquetGroupConverter converter =
              new DrillParquetGroupConverter(
                  mutator, mapWriter.list(name).map(), type.asGroupType(), c, options);
          converters.add(converter);
        }
      } else {
        PrimitiveConverter converter = getConverterForType(name, type.asPrimitiveType());
        converters.add(converter);
      }
    }
  }
 @Override
 public void end() {
   mapWriter.end();
 }
 @Override
 public void start() {
   mapWriter.start();
 }
  private PrimitiveConverter getConverterForType(String name, PrimitiveType type) {

    switch (type.getPrimitiveTypeName()) {
      case INT32:
        {
          if (type.getOriginalType() == null) {
            IntWriter writer =
                type.getRepetition() == Repetition.REPEATED
                    ? mapWriter.list(name).integer()
                    : mapWriter.integer(name);
            return new DrillIntConverter(writer);
          }
          switch (type.getOriginalType()) {
            case DECIMAL:
              {
                ParquetReaderUtility.checkDecimalTypeEnabled(options);
                Decimal9Writer writer =
                    type.getRepetition() == Repetition.REPEATED
                        ? mapWriter.list(name).decimal9()
                        : mapWriter.decimal9(name);
                return new DrillDecimal9Converter(
                    writer,
                    type.getDecimalMetadata().getPrecision(),
                    type.getDecimalMetadata().getScale());
              }
            case DATE:
              {
                DateWriter writer =
                    type.getRepetition() == Repetition.REPEATED
                        ? mapWriter.list(name).date()
                        : mapWriter.date(name);
                return new DrillDateConverter(writer);
              }
            case TIME_MILLIS:
              {
                TimeWriter writer =
                    type.getRepetition() == Repetition.REPEATED
                        ? mapWriter.list(name).time()
                        : mapWriter.time(name);
                return new DrillTimeConverter(writer);
              }
            default:
              {
                throw new UnsupportedOperationException(
                    "Unsupported type: " + type.getOriginalType());
              }
          }
        }
      case INT64:
        {
          if (type.getOriginalType() == null) {
            BigIntWriter writer =
                type.getRepetition() == Repetition.REPEATED
                    ? mapWriter.list(name).bigInt()
                    : mapWriter.bigInt(name);
            return new DrillBigIntConverter(writer);
          }
          switch (type.getOriginalType()) {
            case DECIMAL:
              {
                ParquetReaderUtility.checkDecimalTypeEnabled(options);
                Decimal18Writer writer =
                    type.getRepetition() == Repetition.REPEATED
                        ? mapWriter.list(name).decimal18()
                        : mapWriter.decimal18(name);
                return new DrillDecimal18Converter(
                    writer,
                    type.getDecimalMetadata().getPrecision(),
                    type.getDecimalMetadata().getScale());
              }
            case TIMESTAMP_MILLIS:
              {
                TimeStampWriter writer =
                    type.getRepetition() == Repetition.REPEATED
                        ? mapWriter.list(name).timeStamp()
                        : mapWriter.timeStamp(name);
                return new DrillTimeStampConverter(writer);
              }
            default:
              {
                throw new UnsupportedOperationException(
                    "Unsupported type " + type.getOriginalType());
              }
          }
        }
      case INT96:
        {
          if (type.getOriginalType() == null) {
            VarBinaryWriter writer =
                type.getRepetition() == Repetition.REPEATED
                    ? mapWriter.list(name).varBinary()
                    : mapWriter.varBinary(name);
            return new DrillFixedBinaryToVarbinaryConverter(
                writer,
                ParquetRecordReader.getTypeLengthInBits(type.getPrimitiveTypeName()) / 8,
                mutator.getManagedBuffer());
          }
        }
      case FLOAT:
        {
          Float4Writer writer =
              type.getRepetition() == Repetition.REPEATED
                  ? mapWriter.list(name).float4()
                  : mapWriter.float4(name);
          return new DrillFloat4Converter(writer);
        }
      case DOUBLE:
        {
          Float8Writer writer =
              type.getRepetition() == Repetition.REPEATED
                  ? mapWriter.list(name).float8()
                  : mapWriter.float8(name);
          return new DrillFloat8Converter(writer);
        }
      case BOOLEAN:
        {
          BitWriter writer =
              type.getRepetition() == Repetition.REPEATED
                  ? mapWriter.list(name).bit()
                  : mapWriter.bit(name);
          return new DrillBoolConverter(writer);
        }
      case BINARY:
        {
          if (type.getOriginalType() == null) {
            VarBinaryWriter writer =
                type.getRepetition() == Repetition.REPEATED
                    ? mapWriter.list(name).varBinary()
                    : mapWriter.varBinary(name);
            return new DrillVarBinaryConverter(writer, mutator.getManagedBuffer());
          }
          switch (type.getOriginalType()) {
            case UTF8:
              {
                VarCharWriter writer =
                    type.getRepetition() == Repetition.REPEATED
                        ? mapWriter.list(name).varChar()
                        : mapWriter.varChar(name);
                return new DrillVarCharConverter(writer, mutator.getManagedBuffer());
              }
              // TODO not sure if BINARY/DECIMAL is actually supported
            case DECIMAL:
              {
                ParquetReaderUtility.checkDecimalTypeEnabled(options);
                DecimalMetadata metadata = type.getDecimalMetadata();
                if (metadata.getPrecision() <= 28) {
                  Decimal28SparseWriter writer =
                      type.getRepetition() == Repetition.REPEATED
                          ? mapWriter.list(name).decimal28Sparse()
                          : mapWriter.decimal28Sparse(
                              name, metadata.getScale(), metadata.getPrecision());
                  return new DrillBinaryToDecimal28Converter(
                      writer,
                      metadata.getPrecision(),
                      metadata.getScale(),
                      mutator.getManagedBuffer());
                } else {
                  Decimal38SparseWriter writer =
                      type.getRepetition() == Repetition.REPEATED
                          ? mapWriter.list(name).decimal38Sparse()
                          : mapWriter.decimal38Sparse(
                              name, metadata.getScale(), metadata.getPrecision());
                  return new DrillBinaryToDecimal38Converter(
                      writer,
                      metadata.getPrecision(),
                      metadata.getScale(),
                      mutator.getManagedBuffer());
                }
              }
            default:
              {
                throw new UnsupportedOperationException(
                    "Unsupported type " + type.getOriginalType());
              }
          }
        }
      case FIXED_LEN_BYTE_ARRAY:
        if (type.getOriginalType() == OriginalType.DECIMAL) {
          ParquetReaderUtility.checkDecimalTypeEnabled(options);
          DecimalMetadata metadata = type.getDecimalMetadata();
          if (metadata.getPrecision() <= 28) {
            Decimal28SparseWriter writer =
                type.getRepetition() == Repetition.REPEATED
                    ? mapWriter.list(name).decimal28Sparse()
                    : mapWriter.decimal28Sparse(name, metadata.getScale(), metadata.getPrecision());
            return new DrillBinaryToDecimal28Converter(
                writer, metadata.getPrecision(), metadata.getScale(), mutator.getManagedBuffer());
          } else {
            Decimal38SparseWriter writer =
                type.getRepetition() == Repetition.REPEATED
                    ? mapWriter.list(name).decimal38Sparse()
                    : mapWriter.decimal38Sparse(name, metadata.getScale(), metadata.getPrecision());
            return new DrillBinaryToDecimal38Converter(
                writer, metadata.getPrecision(), metadata.getScale(), mutator.getManagedBuffer());
          }
        } else if (type.getOriginalType() == OriginalType.INTERVAL) {
          IntervalWriter writer =
              type.getRepetition() == Repetition.REPEATED
                  ? mapWriter.list(name).interval()
                  : mapWriter.interval(name);
          return new DrillFixedLengthByteArrayToInterval(writer);

        } else {
          VarBinaryWriter writer =
              type.getRepetition() == Repetition.REPEATED
                  ? mapWriter.list(name).varBinary()
                  : mapWriter.varBinary(name);
          return new DrillFixedBinaryToVarbinaryConverter(
              writer, type.getTypeLength(), mutator.getManagedBuffer());
        }
      default:
        throw new UnsupportedOperationException("Unsupported type: " + type.getPrimitiveTypeName());
    }
  }