@Override
public void reduce(
BytesWritable topkRollupKey, Iterable<BytesWritable> timeSeriesIterable, Context context)
throws IOException, InterruptedException {
TopKRollupPhaseOneMapOutputKey wrapper =
TopKRollupPhaseOneMapOutputKey.fromBytes(topkRollupKey.getBytes());
LOGGER.info(
"DimensionName {} DimensionValue {}",
wrapper.getDimensionName(),
wrapper.getDimensionValue());
MetricTimeSeries aggregateSeries = new MetricTimeSeries(metricSchema);
for (BytesWritable writable : timeSeriesIterable) {
MetricTimeSeries series = MetricTimeSeries.fromBytes(writable.copyBytes(), metricSchema);
aggregateSeries.aggregate(series);
}
Map<String, Long> metricValues = new HashMap<String, Long>();
for (MetricSpec metricSpec : starTreeConfig.getMetrics()) {
metricValues.put(metricSpec.getName(), 0L);
}
for (Long time : aggregateSeries.getTimeWindowSet()) {
for (MetricSpec metricSpec : starTreeConfig.getMetrics()) {
String metricName = metricSpec.getName();
long metricValue = aggregateSeries.get(time, metricName).longValue();
metricValues.put(metricName, metricValues.get(metricName) + metricValue);
}
}
boolean aboveThreshold = true;
for (MetricSpec metricSpec : starTreeConfig.getMetrics()) {
String metricName = metricSpec.getName();
long metricValue = metricValues.get(metricName);
long metricSum = metricSums.get(metricName);
double metricThreshold = metricThresholds.get(metricName);
LOGGER.info("metricValue : {} metricSum : {}", metricValue, metricSum);
if (metricValue < (metricThreshold / 100) * metricSum) {
aboveThreshold = false;
break;
}
}
if (aboveThreshold) {
LOGGER.info("Passed threshold");
valWritable.set(aggregateSeries.toBytes(), 0, aggregateSeries.toBytes().length);
context.write(topkRollupKey, valWritable);
}
}
@Override
public void setup(Context context) throws IOException, InterruptedException {
LOGGER.info("TopKRollupPhaseOneJob.TopKRollupPhaseOneReducer.setup()");
Configuration configuration = context.getConfiguration();
FileSystem fileSystem = FileSystem.get(configuration);
Path configPath = new Path(configuration.get(TOPK_ROLLUP_PHASE1_CONFIG_PATH.toString()));
try {
starTreeConfig = StarTreeConfig.decode(fileSystem.open(configPath));
config = TopKRollupPhaseOneConfig.fromStarTreeConfig(starTreeConfig);
dimensionNames = config.getDimensionNames();
metricTypes = config.getMetricTypes();
metricSchema = new MetricSchema(config.getMetricNames(), metricTypes);
metricThresholds = config.getMetricThresholds();
keyWritable = new BytesWritable();
valWritable = new BytesWritable();
MetricSums metricSumsObj =
OBJECT_MAPPER.readValue(
fileSystem.open(
new Path(configuration.get(TOPK_ROLLUP_PHASE1_METRIC_SUMS_PATH.toString()))),
MetricSums.class);
metricSums = metricSumsObj.getMetricSum();
} catch (Exception e) {
throw new IOException(e);
}
}
@Override
public List<DimensionKey> getDimensionKeys() {
List<DimensionKey> dimensionKeys = new ArrayList<DimensionKey>();
ByteBuffer tmpBuffer = buffer.duplicate();
while (tmpBuffer.position() < tmpBuffer.limit()) {
String[] dimensionValues = new String[config.getDimensions().size()];
for (int i = 0; i < config.getDimensions().size(); i++) {
DimensionSpec dimensionSpec = config.getDimensions().get(i);
Integer valueId = tmpBuffer.getInt();
String dimensionValue = dictionary.getDimensionValue(dimensionSpec.getName(), valueId);
dimensionValues[i] = dimensionValue;
}
dimensionKeys.add(new DimensionKey(dimensionValues));
}
return dimensionKeys;
}
@Override
public void setup(Context context) throws IOException, InterruptedException {
LOGGER.info("TopKRollupPhaseOneJob.TopKRollupPhaseOneMapper.setup()");
Configuration configuration = context.getConfiguration();
FileSystem fileSystem = FileSystem.get(configuration);
Path configPath = new Path(configuration.get(TOPK_ROLLUP_PHASE1_CONFIG_PATH.toString()));
try {
starTreeConfig = StarTreeConfig.decode(fileSystem.open(configPath));
config = TopKRollupPhaseOneConfig.fromStarTreeConfig(starTreeConfig);
dimensionNames = config.getDimensionNames();
keyWritable = new BytesWritable();
valWritable = new BytesWritable();
dimensionNameToIndexMapping = new HashMap<String, Integer>();
for (int i = 0; i < dimensionNames.size(); i++) {
dimensionNameToIndexMapping.put(dimensionNames.get(i), i);
}
} catch (Exception e) {
throw new IOException(e);
}
}
@Override
public Map<DimensionKey, Integer> findMatchingKeys(DimensionKey dimensionKey) {
Map<DimensionKey, Integer> matchingKeys = new HashMap<DimensionKey, Integer>();
int[] translatedKey = dictionary.translate(config.getDimensions(), dimensionKey);
int[] currentKey = new int[config.getDimensions().size()];
int idx = 0;
ByteBuffer tmpBuffer = buffer.duplicate();
while (tmpBuffer.position() < tmpBuffer.limit()) {
boolean matches = true;
for (int i = 0; i < config.getDimensions().size(); i++) {
Integer valueId = tmpBuffer.getInt();
currentKey[i] = valueId;
if (translatedKey[i] != valueId && translatedKey[i] != StarTreeConstants.STAR_VALUE) {
matches = false;
}
}
if (matches) {
matchingKeys.put(dictionary.translate(config.getDimensions(), currentKey), idx);
}
idx++;
}
// If matching keys is empty, use record with least others!
if (matchingKeys.isEmpty()) {
idx = 0;
tmpBuffer.rewind();
int leastNumOthers = config.getDimensions().size() + 1;
int leastOthersIdx = -1;
int[] leastOthersKey = null;
while (tmpBuffer.position() < tmpBuffer.limit()) {
boolean matches = true;
int currentNumOthers = 0;
for (int i = 0; i < config.getDimensions().size(); i++) {
Integer valueId = tmpBuffer.getInt();
currentKey[i] = valueId;
if (translatedKey[i] != valueId
&& valueId != StarTreeConstants.STAR_VALUE
&& valueId != StarTreeConstants.OTHER_VALUE) {
matches = false;
}
if (valueId == StarTreeConstants.OTHER_VALUE) {
currentNumOthers++;
}
}
if (matches && currentNumOthers < leastNumOthers) {
leastOthersKey = Arrays.copyOf(currentKey, currentKey.length);
leastNumOthers = currentNumOthers;
leastOthersIdx = idx;
}
idx++;
}
if (leastOthersKey == null) {
throw new IllegalStateException(
"Could not find alternative dimension combination for " + dimensionKey);
}
matchingKeys.put(
dictionary.translate(config.getDimensions(), leastOthersKey), leastOthersIdx);
}
return matchingKeys;
}