protected static List<ByteArrayRange> getQueryRanges(
final BinnedNumericDataset[] binnedQueries,
final SpaceFillingCurve sfc,
final int maxRanges,
final byte tier) {
final List<ByteArrayRange> queryRanges = new ArrayList<ByteArrayRange>();
int maxRangeDecompositionPerBin = maxRanges;
if ((maxRanges > 1) && (binnedQueries.length > 1)) {
maxRangeDecompositionPerBin =
(int) Math.ceil((double) maxRanges / (double) binnedQueries.length);
}
for (final BinnedNumericDataset binnedQuery : binnedQueries) {
final RangeDecomposition rangeDecomp =
sfc.decomposeRange(binnedQuery, true, maxRangeDecompositionPerBin);
final byte[] tierAndBinId =
ByteArrayUtils.combineArrays(
new byte[] {tier
// we're assuming tiers only go to 127 (the max byte
// value)
},
binnedQuery.getBinId());
for (final ByteArrayRange range : rangeDecomp.getRanges()) {
queryRanges.add(
new ByteArrayRange(
new ByteArrayId(
ByteArrayUtils.combineArrays(tierAndBinId, range.getStart().getBytes())),
new ByteArrayId(
ByteArrayUtils.combineArrays(tierAndBinId, range.getEnd().getBytes()))));
}
}
return queryRanges;
}
protected static List<ByteArrayId> decomposeRangesForEntry(
final BinnedNumericDataset index, final byte tierId, final SpaceFillingCurve sfc) {
final List<ByteArrayId> retVal = new ArrayList<ByteArrayId>();
final byte[] tierAndBinId = ByteArrayUtils.combineArrays(new byte[] {tierId}, index.getBinId());
final RangeDecomposition rangeDecomp = sfc.decomposeRange(index, false, DEFAULT_MAX_RANGES);
// this range does not fit into a single row ID at the lowest
// tier, decompose it
for (final ByteArrayRange range : rangeDecomp.getRanges()) {
final byte[] currentRowId =
Arrays.copyOf(range.getStart().getBytes(), range.getStart().getBytes().length);
retVal.add(new ByteArrayId(ByteArrayUtils.combineArrays(tierAndBinId, currentRowId)));
while (!Arrays.equals(currentRowId, range.getEnd().getBytes())) {
// increment until we reach the end row ID
boolean overflow = !ByteArrayUtils.increment(currentRowId);
if (!overflow) {
retVal.add(new ByteArrayId(ByteArrayUtils.combineArrays(tierAndBinId, currentRowId)));
} else {
// the increment caused an overflow which shouldn't
// ever happen assuming the start row ID is less
// than the end row ID
LOGGER.warn(
"Row IDs overflowed when ingesting data; start of range decomposition must be less than or equal to end of range. This may be because the start of the decomposed range is higher than the end of the range.");
overflow = true;
break;
}
}
}
return retVal;
}
protected static long[] getCoordinatesForId(
final byte[] rowId,
final NumericDimensionDefinition[] baseDefinitions,
final SpaceFillingCurve sfc) {
final SFCIdAndBinInfo sfcIdAndBinInfo = getSFCIdAndBinInfo(rowId, baseDefinitions);
return sfc.getCoordinates(sfcIdAndBinInfo.sfcId);
}
protected static List<ByteArrayId> getRowIdsAtTier(
final BinnedNumericDataset index,
final byte tierId,
final SpaceFillingCurve sfc,
final BigInteger maxEstimatedDuplicateIds,
final int sfcIndex) {
final List<ByteArrayId> retVal = new ArrayList<ByteArrayId>();
final BigInteger rowCount = sfc.getEstimatedIdCount(index);
if (rowCount.equals(BigInteger.ONE)) {
final byte[] tierAndBinId =
ByteArrayUtils.combineArrays(new byte[] {tierId}, index.getBinId());
final double[] maxValues = index.getMaxValuesPerDimension();
retVal.add(new ByteArrayId(ByteArrayUtils.combineArrays(tierAndBinId, sfc.getId(maxValues))));
return retVal;
} else if ((maxEstimatedDuplicateIds == null)
|| (rowCount.compareTo(maxEstimatedDuplicateIds) <= 0)
|| (sfcIndex == 0)) {
return decomposeRangesForEntry(index, tierId, sfc);
}
return null;
}
protected static MultiDimensionalNumericData getRangeForId(
final byte[] rowId,
final NumericDimensionDefinition[] baseDefinitions,
final SpaceFillingCurve sfc) {
final SFCIdAndBinInfo sfcIdAndBinInfo = getSFCIdAndBinInfo(rowId, baseDefinitions);
final MultiDimensionalNumericData numericData = sfc.getRanges(sfcIdAndBinInfo.sfcId);
// now we need to unapply the bins to the data, denormalizing the
// ranges to the native bounds
if (sfcIdAndBinInfo.rowIdOffset > 1) {
final NumericData[] data = numericData.getDataPerDimension();
for (final Entry<Integer, byte[]> entry : sfcIdAndBinInfo.binIds.entrySet()) {
final int dimension = entry.getKey();
final NumericRange range =
baseDefinitions[dimension].getDenormalizedRange(
new BinRange(
entry.getValue(), data[dimension].getMin(), data[dimension].getMax(), false));
data[dimension] = range;
}
return new BasicNumericDataset(data);
}
return numericData;
}