예제 #1
0
    @Override
    public void reduce(
        BytesWritable topkRollupKey, Iterable<BytesWritable> timeSeriesIterable, Context context)
        throws IOException, InterruptedException {

      TopKRollupPhaseOneMapOutputKey wrapper =
          TopKRollupPhaseOneMapOutputKey.fromBytes(topkRollupKey.getBytes());
      LOGGER.info(
          "DimensionName {} DimensionValue {}",
          wrapper.getDimensionName(),
          wrapper.getDimensionValue());

      MetricTimeSeries aggregateSeries = new MetricTimeSeries(metricSchema);
      for (BytesWritable writable : timeSeriesIterable) {
        MetricTimeSeries series = MetricTimeSeries.fromBytes(writable.copyBytes(), metricSchema);
        aggregateSeries.aggregate(series);
      }

      Map<String, Long> metricValues = new HashMap<String, Long>();
      for (MetricSpec metricSpec : starTreeConfig.getMetrics()) {
        metricValues.put(metricSpec.getName(), 0L);
      }
      for (Long time : aggregateSeries.getTimeWindowSet()) {
        for (MetricSpec metricSpec : starTreeConfig.getMetrics()) {
          String metricName = metricSpec.getName();
          long metricValue = aggregateSeries.get(time, metricName).longValue();
          metricValues.put(metricName, metricValues.get(metricName) + metricValue);
        }
      }

      boolean aboveThreshold = true;
      for (MetricSpec metricSpec : starTreeConfig.getMetrics()) {
        String metricName = metricSpec.getName();

        long metricValue = metricValues.get(metricName);
        long metricSum = metricSums.get(metricName);
        double metricThreshold = metricThresholds.get(metricName);

        LOGGER.info("metricValue : {} metricSum : {}", metricValue, metricSum);
        if (metricValue < (metricThreshold / 100) * metricSum) {
          aboveThreshold = false;
          break;
        }
      }

      if (aboveThreshold) {
        LOGGER.info("Passed threshold");
        valWritable.set(aggregateSeries.toBytes(), 0, aggregateSeries.toBytes().length);
        context.write(topkRollupKey, valWritable);
      }
    }
예제 #2
0
    @Override
    public void setup(Context context) throws IOException, InterruptedException {

      LOGGER.info("TopKRollupPhaseOneJob.TopKRollupPhaseOneReducer.setup()");

      Configuration configuration = context.getConfiguration();
      FileSystem fileSystem = FileSystem.get(configuration);
      Path configPath = new Path(configuration.get(TOPK_ROLLUP_PHASE1_CONFIG_PATH.toString()));
      try {
        starTreeConfig = StarTreeConfig.decode(fileSystem.open(configPath));
        config = TopKRollupPhaseOneConfig.fromStarTreeConfig(starTreeConfig);
        dimensionNames = config.getDimensionNames();
        metricTypes = config.getMetricTypes();
        metricSchema = new MetricSchema(config.getMetricNames(), metricTypes);
        metricThresholds = config.getMetricThresholds();
        keyWritable = new BytesWritable();
        valWritable = new BytesWritable();

        MetricSums metricSumsObj =
            OBJECT_MAPPER.readValue(
                fileSystem.open(
                    new Path(configuration.get(TOPK_ROLLUP_PHASE1_METRIC_SUMS_PATH.toString()))),
                MetricSums.class);
        metricSums = metricSumsObj.getMetricSum();

      } catch (Exception e) {
        throw new IOException(e);
      }
    }
예제 #3
0
  @Override
  public List<DimensionKey> getDimensionKeys() {
    List<DimensionKey> dimensionKeys = new ArrayList<DimensionKey>();

    ByteBuffer tmpBuffer = buffer.duplicate();

    while (tmpBuffer.position() < tmpBuffer.limit()) {
      String[] dimensionValues = new String[config.getDimensions().size()];

      for (int i = 0; i < config.getDimensions().size(); i++) {
        DimensionSpec dimensionSpec = config.getDimensions().get(i);
        Integer valueId = tmpBuffer.getInt();
        String dimensionValue = dictionary.getDimensionValue(dimensionSpec.getName(), valueId);
        dimensionValues[i] = dimensionValue;
      }

      dimensionKeys.add(new DimensionKey(dimensionValues));
    }

    return dimensionKeys;
  }
예제 #4
0
    @Override
    public void setup(Context context) throws IOException, InterruptedException {
      LOGGER.info("TopKRollupPhaseOneJob.TopKRollupPhaseOneMapper.setup()");
      Configuration configuration = context.getConfiguration();
      FileSystem fileSystem = FileSystem.get(configuration);
      Path configPath = new Path(configuration.get(TOPK_ROLLUP_PHASE1_CONFIG_PATH.toString()));
      try {
        starTreeConfig = StarTreeConfig.decode(fileSystem.open(configPath));
        config = TopKRollupPhaseOneConfig.fromStarTreeConfig(starTreeConfig);
        dimensionNames = config.getDimensionNames();
        keyWritable = new BytesWritable();
        valWritable = new BytesWritable();
        dimensionNameToIndexMapping = new HashMap<String, Integer>();
        for (int i = 0; i < dimensionNames.size(); i++) {
          dimensionNameToIndexMapping.put(dimensionNames.get(i), i);
        }

      } catch (Exception e) {
        throw new IOException(e);
      }
    }
예제 #5
0
  @Override
  public Map<DimensionKey, Integer> findMatchingKeys(DimensionKey dimensionKey) {
    Map<DimensionKey, Integer> matchingKeys = new HashMap<DimensionKey, Integer>();

    int[] translatedKey = dictionary.translate(config.getDimensions(), dimensionKey);
    int[] currentKey = new int[config.getDimensions().size()];

    int idx = 0;

    ByteBuffer tmpBuffer = buffer.duplicate();

    while (tmpBuffer.position() < tmpBuffer.limit()) {
      boolean matches = true;

      for (int i = 0; i < config.getDimensions().size(); i++) {
        Integer valueId = tmpBuffer.getInt();

        currentKey[i] = valueId;

        if (translatedKey[i] != valueId && translatedKey[i] != StarTreeConstants.STAR_VALUE) {
          matches = false;
        }
      }

      if (matches) {
        matchingKeys.put(dictionary.translate(config.getDimensions(), currentKey), idx);
      }

      idx++;
    }

    // If matching keys is empty, use record with least others!
    if (matchingKeys.isEmpty()) {
      idx = 0;
      tmpBuffer.rewind();

      int leastNumOthers = config.getDimensions().size() + 1;
      int leastOthersIdx = -1;
      int[] leastOthersKey = null;

      while (tmpBuffer.position() < tmpBuffer.limit()) {
        boolean matches = true;
        int currentNumOthers = 0;

        for (int i = 0; i < config.getDimensions().size(); i++) {
          Integer valueId = tmpBuffer.getInt();

          currentKey[i] = valueId;

          if (translatedKey[i] != valueId
              && valueId != StarTreeConstants.STAR_VALUE
              && valueId != StarTreeConstants.OTHER_VALUE) {
            matches = false;
          }

          if (valueId == StarTreeConstants.OTHER_VALUE) {
            currentNumOthers++;
          }
        }

        if (matches && currentNumOthers < leastNumOthers) {
          leastOthersKey = Arrays.copyOf(currentKey, currentKey.length);
          leastNumOthers = currentNumOthers;
          leastOthersIdx = idx;
        }

        idx++;
      }

      if (leastOthersKey == null) {
        throw new IllegalStateException(
            "Could not find alternative dimension combination for " + dimensionKey);
      }

      matchingKeys.put(
          dictionary.translate(config.getDimensions(), leastOthersKey), leastOthersIdx);
    }

    return matchingKeys;
  }