@Setup
  public void setup() {
    Random random = new Random();
    RowExpression[] arguments = new RowExpression[1 + inListCount];
    switch (type) {
      case StandardTypes.BIGINT:
        prestoType = BIGINT;
        for (int i = 1; i <= inListCount; i++) {
          arguments[i] = constant((long) random.nextInt(), BIGINT);
        }
        break;
      case StandardTypes.DOUBLE:
        prestoType = DOUBLE;
        for (int i = 1; i <= inListCount; i++) {
          arguments[i] = constant(random.nextDouble(), DOUBLE);
        }
        break;
      case StandardTypes.VARCHAR:
        prestoType = VARCHAR;
        for (int i = 1; i <= inListCount; i++) {
          arguments[i] = constant(Slices.utf8Slice(Long.toString(random.nextLong())), VARCHAR);
        }
        break;
      default:
        throw new IllegalStateException();
    }

    arguments[0] = field(0, prestoType);
    RowExpression project = field(0, prestoType);

    PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(prestoType));
    for (int i = 0; i < 10_000; i++) {
      pageBuilder.declarePosition();

      switch (type) {
        case StandardTypes.BIGINT:
          BIGINT.writeLong(pageBuilder.getBlockBuilder(0), random.nextInt());
          break;
        case StandardTypes.DOUBLE:
          DOUBLE.writeDouble(pageBuilder.getBlockBuilder(0), random.nextDouble());
          break;
        case StandardTypes.VARCHAR:
          VARCHAR.writeSlice(
              pageBuilder.getBlockBuilder(0), Slices.utf8Slice(Long.toString(random.nextLong())));
          break;
      }
    }
    inputPage = pageBuilder.build();

    RowExpression filter =
        call(
            new Signature(IN, SCALAR, parseTypeSignature(StandardTypes.BOOLEAN)),
            BOOLEAN,
            arguments);

    processor =
        new ExpressionCompiler(MetadataManager.createTestMetadataManager())
            .compilePageProcessor(filter, ImmutableList.of(project))
            .get();
  }
Ejemplo n.º 2
0
  private void parseStringColumn(int column) {
    // don't include column number in message because it causes boxing which is expensive here
    checkArgument(!isPartitionColumn[column], "Column is a partition key");

    loaded[column] = true;

    Object fieldData = rowInspector.getStructFieldData(rowData, structFields[column]);

    if (fieldData == null) {
      nulls[column] = true;
    } else {
      Object fieldValue =
          ((PrimitiveObjectInspector) fieldInspectors[column]).getPrimitiveJavaObject(fieldData);
      checkState(fieldValue != null, "fieldValue should not be null");
      Slice value;
      if (fieldValue instanceof String) {
        value = Slices.utf8Slice((String) fieldValue);
      } else if (fieldValue instanceof byte[]) {
        value = Slices.wrappedBuffer((byte[]) fieldValue);
      } else if (fieldValue instanceof HiveVarchar) {
        value = Slices.utf8Slice(((HiveVarchar) fieldValue).getValue());
      } else {
        throw new IllegalStateException(
            "unsupported string field type: " + fieldValue.getClass().getName());
      }
      Type type = types[column];
      if (isVarcharType(type)) {
        value = truncateToLength(value, type);
      }
      slices[column] = value;
      nulls[column] = false;
    }
  }
Ejemplo n.º 3
0
  private void parseStringColumn(int column) {
    // don't include column number in message because it causes boxing which is expensive here
    checkArgument(!isPartitionColumn[column], "Column is a partition key");

    loaded[column] = true;

    Object fieldData = rowInspector.getStructFieldData(rowData, structFields[column]);

    if (fieldData == null) {
      nulls[column] = true;
    } else if (hiveTypes[column] == HiveType.MAP
        || hiveTypes[column] == HiveType.LIST
        || hiveTypes[column] == HiveType.STRUCT) {
      // temporarily special case MAP, LIST, and STRUCT types as strings
      slices[column] =
          Slices.wrappedBuffer(
              SerDeUtils.getJsonBytes(sessionTimeZone, fieldData, fieldInspectors[column]));
      nulls[column] = false;
    } else {
      Object fieldValue =
          ((PrimitiveObjectInspector) fieldInspectors[column]).getPrimitiveJavaObject(fieldData);
      checkState(fieldValue != null, "fieldValue should not be null");
      if (fieldValue instanceof String) {
        slices[column] = Slices.utf8Slice((String) fieldValue);
      } else if (fieldValue instanceof byte[]) {
        slices[column] = Slices.wrappedBuffer((byte[]) fieldValue);
      } else {
        throw new IllegalStateException(
            "unsupported string field type: " + fieldValue.getClass().getName());
      }
      nulls[column] = false;
    }
  }
Ejemplo n.º 4
0
  @Test
  public void testUtf8Conversion() {
    String s = "apple \u2603 snowman";
    Slice slice = Slices.copiedBuffer(s, UTF_8);

    assertEquals(Slices.utf8Slice(s), slice);
    assertEquals(slice.toStringUtf8(), s);
    assertEquals(Slices.utf8Slice(s).toStringUtf8(), s);
  }
Ejemplo n.º 5
0
 private static Block createMapBlock(int positionCount, Block keyBlock, Block valueBlock) {
   InterleavedBlock interleavedBlock = new InterleavedBlock(new Block[] {keyBlock, valueBlock});
   int[] offsets = new int[positionCount];
   int mapSize = keyBlock.getPositionCount() / positionCount;
   for (int i = 0; i < positionCount; i++) {
     offsets[i] = mapSize * 2 * i;
   }
   return new ArrayBlock(
       interleavedBlock, Slices.wrappedIntArray(offsets), 0, Slices.allocate(positionCount));
 }
Ejemplo n.º 6
0
  private static Slice dateFormat(
      ISOChronology chronology, Locale locale, long timestamp, Slice formatString) {
    DateTimeFormatter formatter =
        DATETIME_FORMATTER_CACHE.get(formatString).withChronology(chronology).withLocale(locale);

    return Slices.copiedBuffer(formatter.print(timestamp), Charsets.UTF_8);
  }
Ejemplo n.º 7
0
  public GroupByIdBlock getGroupIds(Page page) {
    int positionCount = page.getPositionCount();

    int groupIdBlockSize = SINGLE_LONG.getFixedSize() * positionCount;
    BlockBuilder blockBuilder =
        new BlockBuilder(
            SINGLE_LONG, groupIdBlockSize, Slices.allocate(groupIdBlockSize).getOutput());

    // open cursors for group blocks
    BlockCursor[] cursors = new BlockCursor[channels.length];
    for (int i = 0; i < channels.length; i++) {
      cursors[i] = page.getBlock(channels[i]).cursor();
    }

    // use cursors in hash strategy to provide value for "current" row
    hashStrategy.setCurrentRow(cursors);

    for (int position = 0; position < positionCount; position++) {
      for (BlockCursor cursor : cursors) {
        checkState(cursor.advanceNextPosition());
      }

      int groupId = pagePositionToGroupId.get(CURRENT_ROW_ADDRESS);
      if (groupId < 0) {
        groupId = addNewGroup(cursors);
      }
      blockBuilder.append(groupId);
    }
    UncompressedBlock block = blockBuilder.build();
    return new GroupByIdBlock(nextGroupId, block);
  }
Ejemplo n.º 8
0
    public ChannelBuilder(Type type, int blockSize) {
      checkNotNull(type, "type is null");

      this.type = type;
      this.slice = Slices.allocate(blockSize);
      this.sliceOutput = slice.getOutput();
      this.positionOffsets = new IntArrayList(1024);
    }
 public static Slice toJson(Type rowType, ConnectorSession session, Slice row) {
   Object object = rowType.getObjectValue(session, createBlock(rowType, row), 0);
   try {
     return Slices.utf8Slice(OBJECT_MAPPER.get().writeValueAsString(object));
   } catch (JsonProcessingException e) {
     throw Throwables.propagate(e);
   }
 }
 private void testIsFull(VariableWidthBlockBuilder blockBuilder) {
   assertTrue(blockBuilder.isEmpty());
   while (!blockBuilder.isFull()) {
     VARCHAR.writeSlice(blockBuilder, Slices.allocate(VARCHAR_VALUE_SIZE));
   }
   assertEquals(blockBuilder.getPositionCount(), EXPECTED_ENTRY_COUNT);
   assertEquals(blockBuilder.isFull(), true);
 }
Ejemplo n.º 11
0
 @Description("convert ASCII character code to string")
 @ScalarFunction
 @SqlType(VarcharType.class)
 public static Slice chr(@SqlType(BigintType.class) long n) {
   Slice slice = Slices.allocate(1);
   slice.setByte(0, Ints.saturatedCast(n));
   return slice;
 }
Ejemplo n.º 12
0
 private void newBuffer() {
   this.buffer = allocator.allocate(PageFormat.PAGE_HEADER_SIZE + fixedRecordSize);
   this.bufferSlice = Slices.wrappedBuffer(buffer.array(), buffer.offset(), buffer.capacity());
   this.count = 0;
   this.position = PageFormat.PAGE_HEADER_SIZE;
   this.stringReferences.clear();
   this.stringReferenceSize = 0;
 }
Ejemplo n.º 13
0
 private static Block createKeyBlock(int positionCount, List<String> keys) {
   Block keyDictionaryBlock = createSliceArrayBlock(keys);
   int[] keyIds = new int[positionCount * keys.size()];
   for (int i = 0; i < keyIds.length; i++) {
     keyIds[i] = i % keys.size();
   }
   return new DictionaryBlock(
       positionCount * keys.size(), keyDictionaryBlock, Slices.wrappedIntArray(keyIds));
 }
Ejemplo n.º 14
0
  private static Slice formatDatetime(
      ISOChronology chronology, Locale locale, long timestamp, Slice formatString) {
    String pattern = formatString.toString(Charsets.UTF_8);
    DateTimeFormatter formatter =
        DateTimeFormat.forPattern(pattern).withChronology(chronology).withLocale(locale);

    String datetimeString = formatter.print(timestamp);
    return Slices.wrappedBuffer(datetimeString.getBytes(Charsets.UTF_8));
  }
Ejemplo n.º 15
0
 @Description("converts all the alphabets in the string to upper case")
 @ScalarFunction
 @SqlType(VarcharType.class)
 public static Slice upper(@SqlType(VarcharType.class) Slice slice) {
   Slice upper = Slices.allocate(slice.length());
   for (int i = 0; i < slice.length(); i++) {
     upper.setByte(i, Ascii.toUpperCase((char) slice.getByte(i)));
   }
   return upper;
 }
Ejemplo n.º 16
0
 @Description("concatenates given strings")
 @ScalarFunction
 @SqlType(VarcharType.class)
 public static Slice concat(
     @SqlType(VarcharType.class) Slice str1, @SqlType(VarcharType.class) Slice str2) {
   Slice concat = Slices.allocate(str1.length() + str2.length());
   concat.setBytes(0, str1);
   concat.setBytes(str1.length(), str2);
   return concat;
 }
Ejemplo n.º 17
0
 @Description("reverses the given string")
 @ScalarFunction
 @SqlType(VarcharType.class)
 public static Slice reverse(@SqlType(VarcharType.class) Slice slice) {
   Slice reverse = Slices.allocate(slice.length());
   for (int i = 0, j = slice.length() - 1; i < slice.length(); i++, j--) {
     reverse.setByte(j, slice.getByte(i));
   }
   return reverse;
 }
Ejemplo n.º 18
0
  @Test
  public void testToString() {
    assertEquals(Slices.copiedBuffer("apple", UTF_8).toString(UTF_8), "apple");

    for (int size = 0; size < 100; size++) {
      for (int index = 0; index < size; index++) {
        assertToStrings(allocate(size), index);
      }
    }
  }
Ejemplo n.º 19
0
 @Description("greedily replaces occurrences of a pattern with a string")
 @ScalarFunction
 @SqlType(VarcharType.class)
 public static Slice replace(
     @SqlType(VarcharType.class) Slice str,
     @SqlType(VarcharType.class) Slice search,
     @SqlType(VarcharType.class) Slice replace) {
   String replaced =
       str.toString(Charsets.UTF_8)
           .replace(search.toString(Charsets.UTF_8), replace.toString(Charsets.UTF_8));
   return Slices.copiedBuffer(replaced, Charsets.UTF_8);
 }
Ejemplo n.º 20
0
 public static Slice charPartitionKey(String value, String name, Type columnType) {
   Slice partitionKey = trimSpaces(Slices.utf8Slice(value));
   CharType charType = checkType(columnType, CharType.class, "columnType");
   if (SliceUtf8.countCodePoints(partitionKey) > charType.getLength()) {
     throw new PrestoException(
         HIVE_INVALID_PARTITION_VALUE,
         format(
             "Invalid partition value '%s' for %s partition key: %s",
             value, columnType.toString(), name));
   }
   return partitionKey;
 }
  private Page generateZeroPage(List<Type> types, int rowsCount, int fieldLength) {
    byte[] constantBytes = new byte[fieldLength];
    Arrays.fill(constantBytes, (byte) 42);
    Slice constantSlice = Slices.wrappedBuffer(constantBytes);

    Block[] blocks = new Block[types.size()];
    for (int i = 0; i < blocks.length; i++) {
      blocks[i] = createZeroBlock(types.get(i), rowsCount, constantSlice);
    }

    return new Page(rowsCount, blocks);
  }
Ejemplo n.º 22
0
    private static Block createDictionaryValueBlock(int positionCount, int mapSize) {
      double distinctRatio = 0.82;

      int dictionarySize = (int) (positionCount * mapSize * distinctRatio);
      List<String> dictionaryStrings = new ArrayList<>(dictionarySize);
      for (int i = 0; i < dictionarySize; i++) {
        int wordLength = ThreadLocalRandom.current().nextInt(5, 10);
        dictionaryStrings.add(randomString(wordLength));
      }
      Block dictionaryBlock = createSliceArrayBlock(dictionaryStrings);

      int[] keyIds = new int[positionCount * mapSize];
      for (int i = 0; i < keyIds.length; i++) {
        keyIds[i] = ThreadLocalRandom.current().nextInt(0, dictionarySize);
      }
      return new DictionaryBlock(
          positionCount * mapSize, dictionaryBlock, Slices.wrappedIntArray(keyIds));
    }
  @Override
  public Block copyPositions(List<Integer> positions) {
    int finalLength = positions.stream().mapToInt(this::getLength).sum();
    SliceOutput newSlice = Slices.allocate(finalLength).getOutput();
    int[] newOffsets = new int[positions.size() + 1];
    boolean[] newValueIsNull = new boolean[positions.size()];

    for (int i = 0; i < positions.size(); i++) {
      int position = positions.get(i);
      if (isEntryNull(position)) {
        newValueIsNull[i] = true;
      } else {
        newSlice.appendBytes(
            sliceOutput
                .getUnderlyingSlice()
                .getBytes(getPositionOffset(position), getLength(position)));
      }
      newOffsets[i + 1] = newSlice.size();
    }
    return new VariableWidthBlock(positions.size(), newSlice.slice(), newOffsets, newValueIsNull);
  }
Ejemplo n.º 24
0
  @Test
  public void testMemoryMappedReads() throws IOException {
    Path path = Files.createTempFile("longs", null);
    ImmutableList<Long> values = createRandomLongs(20000);

    Slice output = allocate(values.size() * Longs.BYTES);
    for (int i = 0; i < values.size(); i++) {
      output.setLong(i * Longs.BYTES, values.get(i));
    }

    Files.write(path, output.getBytes());

    Slice slice = Slices.mapFileReadOnly(path.toFile());
    for (int i = 0; i < values.size(); i++) {
      long actual = slice.getLong(i * Longs.BYTES);
      long expected = values.get(i);
      assertEquals(actual, expected);
    }

    assertEquals(slice.getBytes(), output.getBytes());
  }
Ejemplo n.º 25
0
  @SuppressWarnings("CharUsedInArithmeticContext")
  private static void assertToStrings(Slice slice, int index) {
    // fill slice with FF
    slice.fill((byte) 0xFF);

    // set and get the value
    char[] chars = new char[(slice.length() - index) / 2];
    for (int i = 0; i < chars.length; i++) {
      chars[i] = (char) ('a' + (i % 26));
    }
    String string = new String(chars);
    Slice value = Slices.copiedBuffer(string, UTF_8);
    slice.setBytes(index, value);
    assertEquals(slice.toString(index, value.length(), UTF_8), string);

    for (int length = 0; length < value.length(); length++) {
      slice.fill((byte) 0xFF);
      slice.setBytes(index, value, 0, length);
      assertEquals(slice.toString(index, length, UTF_8), string.substring(0, length));
    }
  }
Ejemplo n.º 26
0
  public GenericHiveRecordCursor(
      RecordReader<K, V> recordReader,
      long totalBytes,
      Properties splitSchema,
      List<HivePartitionKey> partitionKeys,
      List<HiveColumnHandle> columns,
      DateTimeZone hiveStorageTimeZone,
      DateTimeZone sessionTimeZone) {
    checkNotNull(recordReader, "recordReader is null");
    checkArgument(totalBytes >= 0, "totalBytes is negative");
    checkNotNull(splitSchema, "splitSchema is null");
    checkNotNull(partitionKeys, "partitionKeys is null");
    checkNotNull(columns, "columns is null");
    checkArgument(!columns.isEmpty(), "columns is empty");
    checkNotNull(hiveStorageTimeZone, "hiveStorageTimeZone is null");
    checkNotNull(sessionTimeZone, "sessionTimeZone is null");

    this.recordReader = recordReader;
    this.totalBytes = totalBytes;
    this.key = recordReader.createKey();
    this.value = recordReader.createValue();
    this.hiveStorageTimeZone = hiveStorageTimeZone;
    this.sessionTimeZone = sessionTimeZone;

    this.deserializer = getDeserializer(splitSchema);
    this.rowInspector = getTableObjectInspector(deserializer);

    int size = columns.size();

    String[] names = new String[size];
    this.types = new Type[size];
    this.hiveTypes = new HiveType[size];

    this.structFields = new StructField[size];
    this.fieldInspectors = new ObjectInspector[size];

    this.isPartitionColumn = new boolean[size];

    this.loaded = new boolean[size];
    this.booleans = new boolean[size];
    this.longs = new long[size];
    this.doubles = new double[size];
    this.slices = new Slice[size];
    this.nulls = new boolean[size];

    // initialize data columns
    for (int i = 0; i < columns.size(); i++) {
      HiveColumnHandle column = columns.get(i);

      names[i] = column.getName();
      types[i] = column.getType();
      hiveTypes[i] = column.getHiveType();

      if (!column.isPartitionKey()) {
        StructField field = rowInspector.getStructFieldRef(column.getName());
        structFields[i] = field;
        fieldInspectors[i] = field.getFieldObjectInspector();
      }

      isPartitionColumn[i] = column.isPartitionKey();
    }

    // parse requested partition columns
    Map<String, HivePartitionKey> partitionKeysByName =
        uniqueIndex(partitionKeys, HivePartitionKey.nameGetter());
    for (int columnIndex = 0; columnIndex < columns.size(); columnIndex++) {
      HiveColumnHandle column = columns.get(columnIndex);
      if (column.isPartitionKey()) {
        HivePartitionKey partitionKey = partitionKeysByName.get(column.getName());
        checkArgument(partitionKey != null, "Unknown partition key %s", column.getName());

        byte[] bytes = partitionKey.getValue().getBytes(Charsets.UTF_8);

        Type type = types[columnIndex];
        if (BOOLEAN.equals(type)) {
          if (isTrue(bytes, 0, bytes.length)) {
            booleans[columnIndex] = true;
          } else if (isFalse(bytes, 0, bytes.length)) {
            booleans[columnIndex] = false;
          } else {
            String valueString = new String(bytes, Charsets.UTF_8);
            throw new IllegalArgumentException(
                String.format(
                    "Invalid partition value '%s' for BOOLEAN partition key %s",
                    valueString, names[columnIndex]));
          }
        } else if (BIGINT.equals(type)) {
          if (bytes.length == 0) {
            throw new IllegalArgumentException(
                String.format(
                    "Invalid partition value '' for BIGINT partition key %s", names[columnIndex]));
          }
          longs[columnIndex] = parseLong(bytes, 0, bytes.length);
        } else if (DOUBLE.equals(type)) {
          if (bytes.length == 0) {
            throw new IllegalArgumentException(
                String.format(
                    "Invalid partition value '' for DOUBLE partition key %s", names[columnIndex]));
          }
          doubles[columnIndex] = parseDouble(bytes, 0, bytes.length);
        } else if (VARCHAR.equals(type)) {
          slices[columnIndex] = Slices.wrappedBuffer(Arrays.copyOf(bytes, bytes.length));
        } else {
          throw new UnsupportedOperationException("Unsupported column type: " + type);
        }
      }
    }
  }
Ejemplo n.º 27
0
 protected Slice allocate(int size) {
   return Slices.allocate(size);
 }
Ejemplo n.º 28
0
  private static void serializePrimitive(
      Type type, BlockBuilder builder, Object object, PrimitiveObjectInspector inspector) {
    requireNonNull(builder, "parent builder is null");

    if (object == null) {
      builder.appendNull();
      return;
    }

    switch (inspector.getPrimitiveCategory()) {
      case BOOLEAN:
        BooleanType.BOOLEAN.writeBoolean(builder, ((BooleanObjectInspector) inspector).get(object));
        return;
      case BYTE:
        TinyintType.TINYINT.writeLong(builder, ((ByteObjectInspector) inspector).get(object));
        return;
      case SHORT:
        SmallintType.SMALLINT.writeLong(builder, ((ShortObjectInspector) inspector).get(object));
        return;
      case INT:
        IntegerType.INTEGER.writeLong(builder, ((IntObjectInspector) inspector).get(object));
        return;
      case LONG:
        BigintType.BIGINT.writeLong(builder, ((LongObjectInspector) inspector).get(object));
        return;
      case FLOAT:
        DoubleType.DOUBLE.writeDouble(builder, ((FloatObjectInspector) inspector).get(object));
        return;
      case DOUBLE:
        DoubleType.DOUBLE.writeDouble(builder, ((DoubleObjectInspector) inspector).get(object));
        return;
      case STRING:
        type.writeSlice(
            builder,
            Slices.utf8Slice(((StringObjectInspector) inspector).getPrimitiveJavaObject(object)));
        return;
      case VARCHAR:
        type.writeSlice(
            builder,
            Slices.utf8Slice(
                ((HiveVarcharObjectInspector) inspector)
                    .getPrimitiveJavaObject(object)
                    .getValue()));
        return;
      case CHAR:
        CharType charType = checkType(type, CharType.class, "type");
        HiveChar hiveChar = ((HiveCharObjectInspector) inspector).getPrimitiveJavaObject(object);
        type.writeSlice(
            builder,
            trimSpacesAndTruncateToLength(
                Slices.utf8Slice(hiveChar.getValue()), charType.getLength()));
        return;
      case DATE:
        DateType.DATE.writeLong(builder, formatDateAsLong(object, (DateObjectInspector) inspector));
        return;
      case TIMESTAMP:
        TimestampType.TIMESTAMP.writeLong(
            builder, formatTimestampAsLong(object, (TimestampObjectInspector) inspector));
        return;
      case BINARY:
        VARBINARY.writeSlice(
            builder,
            Slices.wrappedBuffer(
                ((BinaryObjectInspector) inspector).getPrimitiveJavaObject(object)));
        return;
      case DECIMAL:
        DecimalType decimalType = checkType(type, DecimalType.class, "type");
        HiveDecimalWritable hiveDecimal =
            ((HiveDecimalObjectInspector) inspector).getPrimitiveWritableObject(object);
        if (decimalType.isShort()) {
          decimalType.writeLong(
              builder, DecimalUtils.getShortDecimalValue(hiveDecimal, decimalType.getScale()));
        } else {
          decimalType.writeSlice(
              builder, DecimalUtils.getLongDecimalValue(hiveDecimal, decimalType.getScale()));
        }
        return;
    }
    throw new RuntimeException("Unknown primitive type: " + inspector.getPrimitiveCategory());
  }
Ejemplo n.º 29
0
  public static Expression toExpression(Object object, Type type) {
    requireNonNull(type, "type is null");

    if (object instanceof Expression) {
      return (Expression) object;
    }

    if (object == null) {
      if (type.equals(UNKNOWN)) {
        return new NullLiteral();
      }
      return new Cast(new NullLiteral(), type.getTypeSignature().toString(), false, true);
    }

    checkArgument(
        Primitives.wrap(type.getJavaType()).isInstance(object),
        "object.getClass (%s) and type.getJavaType (%s) do not agree",
        object.getClass(),
        type.getJavaType());

    if (type.equals(BIGINT)) {
      return new LongLiteral(object.toString());
    }

    if (type.equals(DOUBLE)) {
      Double value = (Double) object;
      // WARNING: the ORC predicate code depends on NaN and infinity not appearing in a tuple
      // domain, so
      // if you remove this, you will need to update the TupleDomainOrcPredicate
      if (value.isNaN()) {
        return new FunctionCall(new QualifiedName("nan"), ImmutableList.<Expression>of());
      } else if (value.equals(Double.NEGATIVE_INFINITY)) {
        return ArithmeticUnaryExpression.negative(
            new FunctionCall(new QualifiedName("infinity"), ImmutableList.<Expression>of()));
      } else if (value.equals(Double.POSITIVE_INFINITY)) {
        return new FunctionCall(new QualifiedName("infinity"), ImmutableList.<Expression>of());
      } else {
        return new DoubleLiteral(object.toString());
      }
    }

    if (type instanceof VarcharType) {
      if (object instanceof String) {
        object = Slices.utf8Slice((String) object);
      }

      if (object instanceof Slice) {
        Slice value = (Slice) object;
        int length = SliceUtf8.countCodePoints(value);

        if (length == ((VarcharType) type).getLength()) {
          return new StringLiteral(value.toStringUtf8());
        }

        return new Cast(
            new StringLiteral(value.toStringUtf8()), type.getDisplayName(), false, true);
      }

      throw new IllegalArgumentException(
          "object must be instance of Slice or String when type is VARCHAR");
    }

    if (type.equals(BOOLEAN)) {
      return new BooleanLiteral(object.toString());
    }

    if (object instanceof Block) {
      SliceOutput output = new DynamicSliceOutput(((Block) object).getSizeInBytes());
      BlockSerdeUtil.writeBlock(output, (Block) object);
      object = output.slice();
      // This if condition will evaluate to true: object instanceof Slice && !type.equals(VARCHAR)
    }

    if (object instanceof Slice) {
      // HACK: we need to serialize VARBINARY in a format that can be embedded in an expression to
      // be
      // able to encode it in the plan that gets sent to workers.
      // We do this by transforming the in-memory varbinary into a call to
      // from_base64(<base64-encoded value>)
      FunctionCall fromBase64 =
          new FunctionCall(
              new QualifiedName("from_base64"),
              ImmutableList.of(
                  new StringLiteral(VarbinaryFunctions.toBase64((Slice) object).toStringUtf8())));
      Signature signature = FunctionRegistry.getMagicLiteralFunctionSignature(type);
      return new FunctionCall(new QualifiedName(signature.getName()), ImmutableList.of(fromBase64));
    }

    Signature signature = FunctionRegistry.getMagicLiteralFunctionSignature(type);
    Expression rawLiteral = toExpression(object, FunctionRegistry.typeForMagicLiteral(type));

    return new FunctionCall(new QualifiedName(signature.getName()), ImmutableList.of(rawLiteral));
  }
Ejemplo n.º 30
0
  @Override
  public Block readBlock(Type type) throws IOException {
    if (!rowGroupOpen) {
      openRowGroup();
    }

    if (readOffset > 0) {
      if (presentStream != null) {
        // skip ahead the present bit reader, but count the set bits
        // and use this as the skip size for the field readers
        readOffset = presentStream.countBitsSet(readOffset);
      }
      for (StreamReader structField : structFields) {
        structField.prepareNextRead(readOffset);
      }
    }

    List<Type> typeParameters = type.getTypeParameters();

    boolean[] nullVector = new boolean[nextBatchSize];
    Block[] blocks = new Block[typeParameters.size()];
    if (presentStream == null) {
      for (int i = 0; i < typeParameters.size(); i++) {
        StreamReader structField = structFields[i];
        structField.prepareNextRead(nextBatchSize);
        blocks[i] = structField.readBlock(typeParameters.get(i));
      }
    } else {
      int nullValues = presentStream.getUnsetBits(nextBatchSize, nullVector);
      if (nullValues != nextBatchSize) {
        for (int i = 0; i < typeParameters.size(); i++) {
          StreamReader structField = structFields[i];
          structField.prepareNextRead(nextBatchSize - nullValues);
          blocks[i] = structField.readBlock(typeParameters.get(i));
        }
      } else {
        for (int i = 0; i < typeParameters.size(); i++) {
          blocks[i] = typeParameters.get(i).createBlockBuilder(new BlockBuilderStatus(), 0).build();
        }
      }
    }

    // Build offsets for array block (null valued have no positions)
    int[] offsets = new int[nextBatchSize];
    offsets[0] = (nullVector[0] ? 0 : typeParameters.size());
    for (int i = 1; i < nextBatchSize; i++) {
      offsets[i] = offsets[i - 1] + (nullVector[i] ? 0 : typeParameters.size());
    }

    // Struct is represented as an array block holding an interleaved block
    InterleavedBlock interleavedBlock = new InterleavedBlock(blocks);
    ArrayBlock arrayBlock =
        new ArrayBlock(
            interleavedBlock,
            Slices.wrappedIntArray(offsets),
            0,
            Slices.wrappedBooleanArray(nullVector));

    readOffset = 0;
    nextBatchSize = 0;

    return arrayBlock;
  }