@Override public void setInput(Configuration conf, Map<String, Block> input, JsonNode json) throws IOException, InterruptedException { // #1. input block inputBlock = (RubixMemoryBlock) input.get(JsonUtils.getText(json, "inputBlock")); // #2. lookup column String lookupColumn = json.get("lookupColumn").getTextValue(); BlockSchema inputSchema = inputBlock.getProperties().getSchema(); coord2offsets = BlockUtils.generateColumnIndex(inputBlock, lookupColumn); // #3. meta data relation name metaRelationName = new String(JsonUtils.getText(json, "metaRelationName")); matchingMetaBlock = (Block) input.get(metaRelationName); BlockSchema metaBlockSchema = matchingMetaBlock.getProperties().getSchema(); // #4. find indexes for coordinate column names in meta relation's schema String[] coordinateColumns = JsonUtils.asArray(json.get("coordinateColumns")); coordinateColumnIndexes = new int[coordinateColumns.length]; int idx = 0; for (String s : JsonUtils.asArray(json.get("coordinateColumns"))) coordinateColumnIndexes[idx++] = metaBlockSchema.getIndex(s); // #5. find index of identifier column in meta relation's schema identifierColumnName = new String(JsonUtils.getText(json, "identifierColumn")); identifierColumnIndex = metaBlockSchema.getIndex(identifierColumnName); // #6. combine columns ArrayNode combineColumns = (ArrayNode) json.get("combineColumns"); // setup info for sort operator /* * jsonForSort = JsonUtils.cloneNode(json); ((ObjectNode) * jsonForSort).put("sortBy", combineColumns); sortedBlock = new * TupleOperatorBlock(sortOp); */ // setup info for combiner operator jsonForCombine = JsonUtils.createObjectNode(); ((ObjectNode) jsonForCombine).put("pivotBy", combineColumns); ((ObjectNode) jsonForCombine).put("schema", inputSchema.toJson()); combinedBlock = new TupleOperatorBlock(combineOp, null); // setup info for generate operator jsonForGenerate = JsonUtils.createObjectNode(); }
@Override public PostCondition getPostCondition(Map<String, PostCondition> preConditions, JsonNode json) throws PreconditionException { String inputBlockName = JsonUtils.getText(json, "input"); PostCondition inputCondition = preConditions.get(inputBlockName); BlockSchema inputSchema = inputCondition.getSchema(); Map<String, CodeDictionary> dictionaryMap = new HashMap<String, CodeDictionary>(); if (json.has("columns")) { String[] columns = JsonUtils.asArray(json, "columns"); for (String column : columns) dictionaryMap.put(column, new CodeDictionary()); } else { JsonNode dictionary = json.get("dictionary"); // this is inline dictionary Iterator<String> nameIterator = dictionary.getFieldNames(); while (nameIterator.hasNext()) { String name = nameIterator.next(); ArrayNode values = (ArrayNode) dictionary.get(name); CodeDictionary codeDictionary = new CodeDictionary(); for (JsonNode value : values) { codeDictionary.addKey(value.getTextValue()); } dictionaryMap.put(name, codeDictionary); } } int numColumns = inputSchema.getNumColumns(); ColumnType[] columnTypes = new ColumnType[numColumns]; for (int i = 0; i < columnTypes.length; i++) { ColumnType type; final String name = inputSchema.getName(i); if (dictionaryMap.containsKey(name)) { // this column is decoded. Transform schema type = new ColumnType(name, DataType.STRING); } else { // this column is not decoded. Reuse schema type = inputSchema.getColumnType(i); } columnTypes[i] = type; } BlockSchema schema = new BlockSchema(columnTypes); return new PostCondition( schema, inputCondition.getPartitionKeys(), inputCondition.getSortKeys()); }
public String[] getSortKeys() throws IOException, ClassNotFoundException { if (keyData == null) getKeyData(); return JsonUtils.asArray(metadataJson.get("sortKeys")); }