public static File merge(
List<IndexableAdapter> indexes,
final AggregatorFactory[] metricAggs,
File outDir,
ProgressIndicator progress)
throws IOException {
FileUtils.deleteDirectory(outDir);
if (!outDir.mkdirs()) {
throw new ISE("Couldn't make outdir[%s].", outDir);
}
final AggregatorFactory[] lowerCaseMetricAggs = new AggregatorFactory[metricAggs.length];
for (int i = 0; i < metricAggs.length; i++) {
lowerCaseMetricAggs[i] = new ToLowerCaseAggregatorFactory(metricAggs[i]);
}
final List<String> mergedDimensions =
mergeIndexed(
Lists.transform(
indexes,
new Function<IndexableAdapter, Iterable<String>>() {
@Override
public Iterable<String> apply(@Nullable IndexableAdapter input) {
return Iterables.transform(
input.getAvailableDimensions(),
new Function<String, String>() {
@Override
public String apply(@Nullable String input) {
return input.toLowerCase();
}
});
}
}));
final List<String> mergedMetrics =
Lists.transform(
mergeIndexed(
Lists.<Iterable<String>>newArrayList(
FunctionalIterable.create(indexes)
.transform(
new Function<IndexableAdapter, Iterable<String>>() {
@Override
public Iterable<String> apply(@Nullable IndexableAdapter input) {
return Iterables.transform(
input.getAvailableMetrics(),
new Function<String, String>() {
@Override
public String apply(@Nullable String input) {
return input.toLowerCase();
}
});
}
})
.concat(
Arrays.<Iterable<String>>asList(
new AggFactoryStringIndexed(lowerCaseMetricAggs))))),
new Function<String, String>() {
@Override
public String apply(@Nullable String input) {
return input.toLowerCase();
}
});
if (mergedMetrics.size() != lowerCaseMetricAggs.length) {
throw new IAE(
"Bad number of metrics[%d], expected [%d]",
mergedMetrics.size(), lowerCaseMetricAggs.length);
}
final AggregatorFactory[] sortedMetricAggs = new AggregatorFactory[mergedMetrics.size()];
for (int i = 0; i < lowerCaseMetricAggs.length; i++) {
AggregatorFactory metricAgg = lowerCaseMetricAggs[i];
sortedMetricAggs[mergedMetrics.indexOf(metricAgg.getName())] = metricAgg;
}
for (int i = 0; i < mergedMetrics.size(); i++) {
if (!sortedMetricAggs[i].getName().equals(mergedMetrics.get(i))) {
throw new IAE(
"Metric mismatch, index[%d] [%s] != [%s]",
i, lowerCaseMetricAggs[i].getName(), mergedMetrics.get(i));
}
}
Function<ArrayList<Iterable<Rowboat>>, Iterable<Rowboat>> rowMergerFn =
new Function<ArrayList<Iterable<Rowboat>>, Iterable<Rowboat>>() {
@Override
public Iterable<Rowboat> apply(@Nullable ArrayList<Iterable<Rowboat>> boats) {
return CombiningIterable.create(
new MergeIterable<Rowboat>(Ordering.<Rowboat>natural().nullsFirst(), boats),
Ordering.<Rowboat>natural().nullsFirst(),
new RowboatMergeFunction(sortedMetricAggs));
}
};
return makeIndexFiles(indexes, outDir, progress, mergedDimensions, mergedMetrics, rowMergerFn);
}
