protected static List<ByteArrayId> decomposeRangesForEntry(
final BinnedNumericDataset index, final byte tierId, final SpaceFillingCurve sfc) {
final List<ByteArrayId> retVal = new ArrayList<ByteArrayId>();
final byte[] tierAndBinId = ByteArrayUtils.combineArrays(new byte[] {tierId}, index.getBinId());
final RangeDecomposition rangeDecomp = sfc.decomposeRange(index, false, DEFAULT_MAX_RANGES);
// this range does not fit into a single row ID at the lowest
// tier, decompose it
for (final ByteArrayRange range : rangeDecomp.getRanges()) {
final byte[] currentRowId =
Arrays.copyOf(range.getStart().getBytes(), range.getStart().getBytes().length);
retVal.add(new ByteArrayId(ByteArrayUtils.combineArrays(tierAndBinId, currentRowId)));
while (!Arrays.equals(currentRowId, range.getEnd().getBytes())) {
// increment until we reach the end row ID
boolean overflow = !ByteArrayUtils.increment(currentRowId);
if (!overflow) {
retVal.add(new ByteArrayId(ByteArrayUtils.combineArrays(tierAndBinId, currentRowId)));
} else {
// the increment caused an overflow which shouldn't
// ever happen assuming the start row ID is less
// than the end row ID
LOGGER.warn(
"Row IDs overflowed when ingesting data; start of range decomposition must be less than or equal to end of range. This may be because the start of the decomposed range is higher than the end of the range.");
overflow = true;
break;
}
}
}
return retVal;
}
protected static List<ByteArrayRange> getQueryRanges(
final BinnedNumericDataset[] binnedQueries,
final SpaceFillingCurve sfc,
final int maxRanges,
final byte tier) {
final List<ByteArrayRange> queryRanges = new ArrayList<ByteArrayRange>();
int maxRangeDecompositionPerBin = maxRanges;
if ((maxRanges > 1) && (binnedQueries.length > 1)) {
maxRangeDecompositionPerBin =
(int) Math.ceil((double) maxRanges / (double) binnedQueries.length);
}
for (final BinnedNumericDataset binnedQuery : binnedQueries) {
final RangeDecomposition rangeDecomp =
sfc.decomposeRange(binnedQuery, true, maxRangeDecompositionPerBin);
final byte[] tierAndBinId =
ByteArrayUtils.combineArrays(
new byte[] {tier
// we're assuming tiers only go to 127 (the max byte
// value)
},
binnedQuery.getBinId());
for (final ByteArrayRange range : rangeDecomp.getRanges()) {
queryRanges.add(
new ByteArrayRange(
new ByteArrayId(
ByteArrayUtils.combineArrays(tierAndBinId, range.getStart().getBytes())),
new ByteArrayId(
ByteArrayUtils.combineArrays(tierAndBinId, range.getEnd().getBytes()))));
}
}
return queryRanges;
}
protected static List<ByteArrayId> getRowIdsAtTier(
final BinnedNumericDataset index,
final byte tierId,
final SpaceFillingCurve sfc,
final BigInteger maxEstimatedDuplicateIds,
final int sfcIndex) {
final List<ByteArrayId> retVal = new ArrayList<ByteArrayId>();
final BigInteger rowCount = sfc.getEstimatedIdCount(index);
if (rowCount.equals(BigInteger.ONE)) {
final byte[] tierAndBinId =
ByteArrayUtils.combineArrays(new byte[] {tierId}, index.getBinId());
final double[] maxValues = index.getMaxValuesPerDimension();
retVal.add(new ByteArrayId(ByteArrayUtils.combineArrays(tierAndBinId, sfc.getId(maxValues))));
return retVal;
} else if ((maxEstimatedDuplicateIds == null)
|| (rowCount.compareTo(maxEstimatedDuplicateIds) <= 0)
|| (sfcIndex == 0)) {
return decomposeRangesForEntry(index, tierId, sfc);
}
return null;
}