@Override
  public void setInput(Configuration conf, Map<String, Block> input, JsonNode json)
      throws IOException, InterruptedException {
    // #1. input block
    inputBlock = (RubixMemoryBlock) input.get(JsonUtils.getText(json, "inputBlock"));

    // #2. lookup column
    String lookupColumn = json.get("lookupColumn").getTextValue();
    BlockSchema inputSchema = inputBlock.getProperties().getSchema();

    coord2offsets = BlockUtils.generateColumnIndex(inputBlock, lookupColumn);

    // #3. meta data relation name
    metaRelationName = new String(JsonUtils.getText(json, "metaRelationName"));
    matchingMetaBlock = (Block) input.get(metaRelationName);
    BlockSchema metaBlockSchema = matchingMetaBlock.getProperties().getSchema();

    // #4. find indexes for coordinate column names in meta relation's schema
    String[] coordinateColumns = JsonUtils.asArray(json.get("coordinateColumns"));
    coordinateColumnIndexes = new int[coordinateColumns.length];
    int idx = 0;
    for (String s : JsonUtils.asArray(json.get("coordinateColumns")))
      coordinateColumnIndexes[idx++] = metaBlockSchema.getIndex(s);

    // #5. find index of identifier column in meta relation's schema
    identifierColumnName = new String(JsonUtils.getText(json, "identifierColumn"));
    identifierColumnIndex = metaBlockSchema.getIndex(identifierColumnName);

    // #6. combine columns
    ArrayNode combineColumns = (ArrayNode) json.get("combineColumns");

    // setup info for sort operator
    /*
     * jsonForSort = JsonUtils.cloneNode(json); ((ObjectNode)
     * jsonForSort).put("sortBy", combineColumns); sortedBlock = new
     * TupleOperatorBlock(sortOp);
     */

    // setup info for combiner operator
    jsonForCombine = JsonUtils.createObjectNode();
    ((ObjectNode) jsonForCombine).put("pivotBy", combineColumns);
    ((ObjectNode) jsonForCombine).put("schema", inputSchema.toJson());
    combinedBlock = new TupleOperatorBlock(combineOp, null);

    // setup info for generate operator
    jsonForGenerate = JsonUtils.createObjectNode();
  }
  @Override
  public PostCondition getPostCondition(Map<String, PostCondition> preConditions, JsonNode json)
      throws PreconditionException {
    String inputBlockName = JsonUtils.getText(json, "input");
    PostCondition inputCondition = preConditions.get(inputBlockName);
    BlockSchema inputSchema = inputCondition.getSchema();

    Map<String, CodeDictionary> dictionaryMap = new HashMap<String, CodeDictionary>();
    if (json.has("columns")) {
      String[] columns = JsonUtils.asArray(json, "columns");
      for (String column : columns) dictionaryMap.put(column, new CodeDictionary());
    } else {
      JsonNode dictionary = json.get("dictionary");
      // this is inline dictionary
      Iterator<String> nameIterator = dictionary.getFieldNames();
      while (nameIterator.hasNext()) {
        String name = nameIterator.next();
        ArrayNode values = (ArrayNode) dictionary.get(name);
        CodeDictionary codeDictionary = new CodeDictionary();
        for (JsonNode value : values) {
          codeDictionary.addKey(value.getTextValue());
        }
        dictionaryMap.put(name, codeDictionary);
      }
    }

    int numColumns = inputSchema.getNumColumns();

    ColumnType[] columnTypes = new ColumnType[numColumns];

    for (int i = 0; i < columnTypes.length; i++) {
      ColumnType type;
      final String name = inputSchema.getName(i);

      if (dictionaryMap.containsKey(name)) {
        // this column is decoded. Transform schema
        type = new ColumnType(name, DataType.STRING);
      } else {
        // this column is not decoded. Reuse schema
        type = inputSchema.getColumnType(i);
      }

      columnTypes[i] = type;
    }

    BlockSchema schema = new BlockSchema(columnTypes);

    return new PostCondition(
        schema, inputCondition.getPartitionKeys(), inputCondition.getSortKeys());
  }
Exemplo n.º 3
0
  public String[] getSortKeys() throws IOException, ClassNotFoundException {
    if (keyData == null) getKeyData();

    return JsonUtils.asArray(metadataJson.get("sortKeys"));
  }