コード例 #1
0
 @Override
 protected void process(Record record, SingleLaneBatchMaker batchMaker) throws StageException {
   Field field = record.get(fieldPath);
   String[] splits = null;
   ErrorCode error = null;
   if (field == null || field.getValue() == null) {
     error = Errors.SPLITTER_01;
   } else {
     String str;
     try {
       str = field.getValueAsString();
     } catch (IllegalArgumentException e) {
       throw new OnRecordErrorException(Errors.SPLITTER_04, fieldPath, field.getType().name());
     }
     splits = str.split(separatorStr, fieldPaths.length);
     if (splits.length < fieldPaths.length) {
       error = Errors.SPLITTER_02;
     }
   }
   if (error == null || onStagePreConditionFailure == OnStagePreConditionFailure.CONTINUE) {
     for (int i = 0; i < fieldPaths.length; i++) {
       try {
         if (splits != null && splits.length > i) {
           record.set(fieldPaths[i], Field.create(splits[i]));
         } else {
           record.set(fieldPaths[i], Field.create(Field.Type.STRING, null));
         }
       } catch (IllegalArgumentException e) {
         throw new OnRecordErrorException(
             Errors.SPLITTER_05, fieldPath, record.getHeader().getSourceId(), e.toString());
       }
     }
     if (removeUnsplitValue) {
       record.delete(fieldPath);
     }
     batchMaker.addRecord(record);
   } else {
     throw new OnRecordErrorException(error, record.getHeader().getSourceId(), fieldPath);
   }
 }
コード例 #2
0
  @Override
  protected void process(Record record, SingleLaneBatchMaker batchMaker) throws StageException {
    Set<String> fieldPaths = record.getFieldPaths();
    List<String> list;
    switch (filterOperation) {
      case REMOVE:
        list = new ArrayList<>();
        for (String field : fields) {
          List<String> matchingFieldPaths = FieldRegexUtil.getMatchingFieldPaths(field, fieldPaths);
          list.addAll(matchingFieldPaths);
        }
        break;
      case KEEP:
        // Algorithm:
        // - Get all possible field paths in the record
        //
        // - Remove arguments fields which must be retained, its parent fields and the child fields
        // from above set
        //   (Account for presence of wild card characters while doing so) The remaining set of
        // fields is what must be
        //   removed from the record.
        //
        // - Sort this set before deleting fields. Last element of a list must be removed first.

        Set<String> fieldsToRemove = new HashSet<>();
        // List all the possible field paths in this record
        fieldsToRemove.addAll(fieldPaths);

        for (String field : fields) {
          // Keep parent fields

          // get the parent fieldPaths for each of the fields to keep
          List<String> parentFieldPaths = getParentFields(field);
          // remove parent paths from the fieldsToRemove set
          // Note that parent names could contain wild card characters
          for (String parentField : parentFieldPaths) {
            List<String> matchingFieldPaths =
                FieldRegexUtil.getMatchingFieldPaths(parentField, fieldPaths);
            fieldsToRemove.removeAll(matchingFieldPaths);
          }

          // Keep the field itself

          // remove the field path itself from the fieldsToRemove set
          // Consider wild card characters
          List<String> matchingFieldPaths = FieldRegexUtil.getMatchingFieldPaths(field, fieldPaths);
          fieldsToRemove.removeAll(matchingFieldPaths);

          // Keep the children of the field

          // For each of the fieldPaths that match the argument field path, generate all the child
          // paths
          List<String> childFieldsToRemove = new ArrayList<>();
          for (String matchingFieldPath : matchingFieldPaths) {
            for (String fieldToRemove : fieldsToRemove) {
              if (fieldToRemove.startsWith(matchingFieldPath)) {
                childFieldsToRemove.add(fieldToRemove);
              }
            }
          }
          fieldsToRemove.removeAll(childFieldsToRemove);
        }
        list = new ArrayList<>(fieldsToRemove);
        break;
      default:
        throw new IllegalStateException(
            Utils.format("Unexpected Filter Operation '{}'", filterOperation.name()));
    }
    Collections.sort(list);
    for (int i = list.size() - 1; i >= 0; i--) {
      record.delete(list.get(i));
    }
    batchMaker.addRecord(record);
  }