Exemplo n.º 1
0
  /** Converts a raw row into its (possibly partial) dimension and complete metric values */
  private StarTreeTableRow extractValues(GenericRow row) {
    List<Integer> dimensions = new ArrayList<Integer>();
    for (String dimensionName : schema.getDimensionNames()) {
      Integer valueId;
      if (schema.getFieldSpecFor(dimensionName).isSingleValueField()
          && !starTreeIndexSpec.getExcludedDimensions().contains(dimensionName)) {
        Object value = row.getValue(dimensionName);
        valueId = dictionaryCreatorMap.get(dimensionName).indexOfSV(value);
      } else {
        // Multi-value fields are not supported - always ALL
        valueId = V1Constants.STARTREE_ALL_NUMBER.intValue();
      }

      dimensions.add(valueId);
    }

    List<Number> metrics = new ArrayList<Number>(schema.getMetricNames().size());
    for (MetricFieldSpec metricFieldSpec : schema.getMetricFieldSpecs()) {
      Object value = row.getValue(metricFieldSpec.getName());
      switch (metricFieldSpec.getDataType()) {
        case INT:
          metrics.add((Integer) value);
          break;
        case LONG:
          metrics.add((Long) value);
          break;
        case DOUBLE:
          metrics.add((Double) value);
          break;
        case FLOAT:
          metrics.add((Float) value);
          break;
        default:
          throw new IllegalStateException("Unsupported data type " + metricFieldSpec.getDataType());
      }
    }

    return new StarTreeTableRow(dimensions, metrics);
  }
Exemplo n.º 2
0
  @Override
  public void indexRow(GenericRow row) {
    // Find matching leaves in StarTree for row
    currentMatchingNodes.clear();
    StarTreeTableRow tableRow = extractValues(row);
    findMatchingLeaves(starTreeBuilder.getTree(), tableRow.getDimensions(), currentMatchingNodes);

    // Only write the raw value, maintaining sort order (we will write aggregates when sealing)
    for (StarTreeIndexNode node : currentMatchingNodes) {
      Map<Integer, Integer> pathValues = node.getPathValues();
      if (!pathValues.containsValue(StarTreeIndexNode.all())) {
        StarTreeTableRange range = starTreeBuilder.getDocumentIdRange(node.getNodeId());
        StarTreeTable subTable =
            starTreeBuilder.getTable().view(range.getStartDocumentId(), range.getDocumentCount());

        Integer nextMatchingDocumentId =
            starTreeBuilder.getNextDocumentId(tableRow.getDimensions());
        if (nextMatchingDocumentId == null) {
          throw new IllegalStateException("Could not assign document ID for row " + tableRow);
        }

        // Write using that document ID to all columns
        for (final String column : dictionaryCreatorMap.keySet()) {
          Object columnValueToIndex = row.getValue(column);
          if (schema.getFieldSpecFor(column).isSingleValueField()) {
            System.out.println(column + ": " + columnValueToIndex);
            int dictionaryIndex = dictionaryCreatorMap.get(column).indexOfSV(columnValueToIndex);
            ((SingleValueForwardIndexCreator) forwardIndexCreatorMap.get(column))
                .index(nextMatchingDocumentId, dictionaryIndex);
            if (config.createInvertedIndexEnabled()) {
              invertedIndexCreatorMap
                  .get(column)
                  .add(nextMatchingDocumentId, (Object) dictionaryIndex);
            }
          } else {
            int[] dictionaryIndex = dictionaryCreatorMap.get(column).indexOfMV(columnValueToIndex);
            ((MultiValueForwardIndexCreator) forwardIndexCreatorMap.get(column))
                .index(nextMatchingDocumentId, dictionaryIndex);
            if (config.createInvertedIndexEnabled()) {
              invertedIndexCreatorMap.get(column).add(nextMatchingDocumentId, dictionaryIndex);
            }
          }
        }
      }
    }
  }
  @Override
  public void indexRow(GenericRow row) {
    for (final String column : dictionaryCreatorMap.keySet()) {

      Object columnValueToIndex = row.getValue(column);
      Object dictionaryIndex;
      if (dictionaryCache.get(column).containsKey(columnValueToIndex)) {
        dictionaryIndex = dictionaryCache.get(column).get(columnValueToIndex);
      } else {
        dictionaryIndex = dictionaryCreatorMap.get(column).indexOf(columnValueToIndex);
        dictionaryCache.get(column).put(columnValueToIndex, dictionaryIndex);
      }
      forwardIndexCreatorMap.get(column).index(docIdCounter, dictionaryIndex);
      if (config.createInvertedIndexEnabled()) {
        invertedIndexCreatorMap.get(column).add(docIdCounter, dictionaryIndex);
      }
    }
    docIdCounter++;
  }