示例#1
0
文件: Keys.java 项目: icklerly/flink
    /** Create String-based (nested) field expression keys on a composite type. */
    public ExpressionKeys(String[] keyExpressions, TypeInformation<T> type) {
      Preconditions.checkNotNull(keyExpressions, "Field expression cannot be null.");

      this.keyFields = new ArrayList<>(keyExpressions.length);

      if (type instanceof CompositeType) {
        CompositeType<T> cType = (CompositeType<T>) type;

        // extract the keys on their flat position
        for (String keyExpr : keyExpressions) {
          if (keyExpr == null) {
            throw new InvalidProgramException("Expression key may not be null.");
          }
          // strip off whitespace
          keyExpr = keyExpr.trim();

          List<FlatFieldDescriptor> flatFields = cType.getFlatFields(keyExpr);

          if (flatFields.size() == 0) {
            throw new InvalidProgramException(
                "Unable to extract key from expression '" + keyExpr + "' on key " + cType);
          }
          // check if all nested fields can be used as keys
          for (FlatFieldDescriptor field : flatFields) {
            if (!field.getType().isKeyType()) {
              throw new InvalidProgramException(
                  "This type (" + field.getType() + ") cannot be used as key.");
            }
          }
          // add flat fields to key fields
          keyFields.addAll(flatFields);
        }
      } else {
        if (!type.isKeyType()) {
          throw new InvalidProgramException("This type (" + type + ") cannot be used as key.");
        }

        // check that all key expressions are valid
        for (String keyExpr : keyExpressions) {
          if (keyExpr == null) {
            throw new InvalidProgramException("Expression key may not be null.");
          }
          // strip off whitespace
          keyExpr = keyExpr.trim();
          // check that full type is addressed
          if (!(SELECT_ALL_CHAR.equals(keyExpr) || SELECT_ALL_CHAR_SCALA.equals(keyExpr))) {
            throw new InvalidProgramException(
                "Field expression must be equal to '"
                    + SELECT_ALL_CHAR
                    + "' or '"
                    + SELECT_ALL_CHAR_SCALA
                    + "' for non-composite types.");
          }
          // add full type as key
          keyFields.add(new FlatFieldDescriptor(0, type));
        }
      }
    }
 @Override
 protected TypeComparator<PojoContainingTuple> createComparator(boolean ascending) {
   Assert.assertTrue(type instanceof CompositeType);
   CompositeType<PojoContainingTuple> cType = (CompositeType<PojoContainingTuple>) type;
   ExpressionKeys<PojoContainingTuple> keys =
       new ExpressionKeys<PojoContainingTuple>(new String[] {"theTuple.*"}, cType);
   boolean[] orders = new boolean[keys.getNumberOfKeyFields()];
   Arrays.fill(orders, ascending);
   return cType.createComparator(
       keys.computeLogicalKeyPositions(), orders, 0, new ExecutionConfig());
 }
示例#3
0
文件: Keys.java 项目: icklerly/flink
    /** Create int-based (non-nested) field position keys on a tuple type. */
    public ExpressionKeys(int[] keyPositions, TypeInformation<T> type, boolean allowEmpty) {

      if (!type.isTupleType() || !(type instanceof CompositeType)) {
        throw new InvalidProgramException(
            "Specifying keys via field positions is only valid "
                + "for tuple data types. Type: "
                + type);
      }
      if (type.getArity() == 0) {
        throw new InvalidProgramException(
            "Tuple size must be greater than 0. Size: " + type.getArity());
      }
      if (!allowEmpty && (keyPositions == null || keyPositions.length == 0)) {
        throw new IllegalArgumentException("The grouping fields must not be empty.");
      }

      this.keyFields = new ArrayList<>();

      if (keyPositions == null || keyPositions.length == 0) {
        // use all tuple fields as key fields
        keyPositions = createIncrIntArray(type.getArity());
      } else {
        rangeCheckFields(keyPositions, type.getArity() - 1);
      }
      Preconditions.checkArgument(
          keyPositions.length > 0, "Grouping fields can not be empty at this point");

      // extract key field types
      CompositeType<T> cType = (CompositeType<T>) type;
      this.keyFields = new ArrayList<>(type.getTotalFields());

      // for each key position, find all (nested) field types
      String[] fieldNames = cType.getFieldNames();
      ArrayList<FlatFieldDescriptor> tmpList = new ArrayList<>();
      for (int keyPos : keyPositions) {
        tmpList.clear();
        // get all flat fields
        cType.getFlatFields(fieldNames[keyPos], 0, tmpList);
        // check if fields are of key type
        for (FlatFieldDescriptor ffd : tmpList) {
          if (!ffd.getType().isKeyType()) {
            throw new InvalidProgramException(
                "This type (" + ffd.getType() + ") cannot be used as key.");
          }
        }
        this.keyFields.addAll(tmpList);
      }
    }
示例#4
0
    /** Create ExpressionKeys from String-expressions */
    public ExpressionKeys(String[] expressionsIn, TypeInformation<T> type) {
      Preconditions.checkNotNull(expressionsIn, "Field expression cannot be null.");

      if (type instanceof AtomicType) {
        if (!type.isKeyType()) {
          throw new InvalidProgramException("This type (" + type + ") cannot be used as key.");
        } else if (expressionsIn.length != 1
            || !(Keys.ExpressionKeys.SELECT_ALL_CHAR.equals(expressionsIn[0])
                || Keys.ExpressionKeys.SELECT_ALL_CHAR_SCALA.equals(expressionsIn[0]))) {
          throw new InvalidProgramException(
              "Field expression for atomic type must be equal to '*' or '_'.");
        }

        keyFields = new ArrayList<>(1);
        keyFields.add(new FlatFieldDescriptor(0, type));
      } else {
        CompositeType<T> cType = (CompositeType<T>) type;

        String[] expressions = removeDuplicates(expressionsIn);
        if (expressionsIn.length != expressions.length) {
          LOG.warn("The key expressions contained duplicates. They are now unique");
        }
        // extract the keys on their flat position
        keyFields = new ArrayList<>(expressions.length);
        for (String expression : expressions) {
          List<FlatFieldDescriptor> keys =
              cType.getFlatFields(expression); // use separate list to do a size check
          for (FlatFieldDescriptor key : keys) {
            TypeInformation<?> keyType = key.getType();
            if (!keyType.isKeyType()) {
              throw new InvalidProgramException(
                  "This type (" + key.getType() + ") cannot be used as key.");
            }
            if (!(keyType instanceof AtomicType || keyType instanceof CompositeType)) {
              throw new InvalidProgramException(
                  "Field type is neither CompositeType nor AtomicType: " + keyType);
            }
          }
          if (keys.size() == 0) {
            throw new InvalidProgramException(
                "Unable to extract key from expression '" + expression + "' on key " + cType);
          }
          keyFields.addAll(keys);
        }
      }
    }
  @Override
  public void getFlatFields(String fieldExpression, int offset, List<FlatFieldDescriptor> result) {

    Matcher matcher = PATTERN_NESTED_FIELDS_WILDCARD.matcher(fieldExpression);
    if (!matcher.matches()) {
      throw new InvalidFieldReferenceException(
          "Invalid POJO field reference \"" + fieldExpression + "\".");
    }

    String field = matcher.group(0);
    if (field.equals(ExpressionKeys.SELECT_ALL_CHAR)
        || field.equals(ExpressionKeys.SELECT_ALL_CHAR_SCALA)) {
      // handle select all
      int keyPosition = 0;
      for (PojoField pField : fields) {
        if (pField.type instanceof CompositeType) {
          CompositeType<?> cType = (CompositeType<?>) pField.type;
          cType.getFlatFields(
              String.valueOf(ExpressionKeys.SELECT_ALL_CHAR), offset + keyPosition, result);
          keyPosition += cType.getTotalFields() - 1;
        } else {
          result.add(
              new NamedFlatFieldDescriptor(
                  pField.field.getName(), offset + keyPosition, pField.type));
        }
        keyPosition++;
      }
      return;
    } else {
      field = matcher.group(1);
    }

    // get field
    int fieldPos = -1;
    TypeInformation<?> fieldType = null;
    for (int i = 0; i < fields.length; i++) {
      if (fields[i].field.getName().equals(field)) {
        fieldPos = i;
        fieldType = fields[i].type;
        break;
      }
    }
    if (fieldPos == -1) {
      throw new InvalidFieldReferenceException(
          "Unable to find field \"" + field + "\" in type " + this + ".");
    }
    String tail = matcher.group(3);
    if (tail == null) {
      if (fieldType instanceof CompositeType) {
        // forward offset
        for (int i = 0; i < fieldPos; i++) {
          offset += this.getTypeAt(i).getTotalFields();
        }
        // add all fields of composite type
        ((CompositeType<?>) fieldType).getFlatFields("*", offset, result);
        return;
      } else {
        // we found the field to add
        // compute flat field position by adding skipped fields
        int flatFieldPos = offset;
        for (int i = 0; i < fieldPos; i++) {
          flatFieldPos += this.getTypeAt(i).getTotalFields();
        }
        result.add(new FlatFieldDescriptor(flatFieldPos, fieldType));
        // nothing left to do
        return;
      }
    } else {
      if (fieldType instanceof CompositeType<?>) {
        // forward offset
        for (int i = 0; i < fieldPos; i++) {
          offset += this.getTypeAt(i).getTotalFields();
        }
        ((CompositeType<?>) fieldType).getFlatFields(tail, offset, result);
        // nothing left to do
        return;
      } else {
        throw new InvalidFieldReferenceException(
            "Nested field expression \""
                + tail
                + "\" not possible on atomic type "
                + fieldType
                + ".");
      }
    }
  }