private static RowRangeHistogramStatistics<?> getRangeStats(
final PrimaryIndex index,
final List<DataAdapter<Object>> adapters,
final AdapterStore adapterStore,
final DataStatisticsStore store,
final String[] authorizations) {
RowRangeHistogramStatistics<?> singleStats = null;
for (final DataAdapter<?> adapter : adapters) {
final RowRangeHistogramStatistics<?> rowStat =
(RowRangeHistogramStatistics<?>)
store.getDataStatistics(
adapter.getAdapterId(),
RowRangeHistogramStatistics.composeId(index.getId()),
authorizations);
if (singleStats == null) {
singleStats = rowStat;
} else {
singleStats.merge(rowStat);
}
}
return singleStats;
}
private void runtest() throws IOException {
final PrimaryIndex index = new SpatialDimensionalityTypeProvider().createPrimaryIndex();
final WritableDataAdapter<TestGeometry> adapter = new TestGeometryAdapter();
final Geometry testGeoFilter =
factory.createPolygon(
new Coordinate[] {
new Coordinate(24, 33),
new Coordinate(28, 33),
new Coordinate(28, 31),
new Coordinate(24, 31),
new Coordinate(24, 33)
});
ByteArrayId rowId0, rowId1;
try (IndexWriter indexWriter =
mockDataStore.createIndexWriter(index, DataStoreUtils.DEFAULT_VISIBILITY)) {
rowId0 =
indexWriter
.write(
adapter,
new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt"),
visWriterAAA)
.get(0);
rowId1 =
indexWriter
.write(
adapter,
new TestGeometry(factory.createPoint(new Coordinate(26, 32)), "test_pt_1"),
visWriterAAA)
.get(0);
indexWriter
.write(
adapter,
new TestGeometry(factory.createPoint(new Coordinate(27, 32)), "test_pt_2"),
visWriterBBB)
.get(0);
}
final SpatialQuery query = new SpatialQuery(testGeoFilter);
try (CloseableIterator<?> it1 =
mockDataStore.query(
new QueryOptions(adapter, index, -1, null, new String[] {"aaa", "bbb"}), query)) {
int count = 0;
while (it1.hasNext()) {
it1.next();
count++;
}
assertEquals(3, count);
}
CountDataStatistics<?> countStats =
(CountDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), CountDataStatistics.STATS_ID, "aaa", "bbb");
assertEquals(3, countStats.getCount());
countStats =
(CountDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), CountDataStatistics.STATS_ID, "aaa");
assertEquals(2, countStats.getCount());
countStats =
(CountDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), CountDataStatistics.STATS_ID, "bbb");
assertEquals(1, countStats.getCount());
BoundingBoxDataStatistics<?> bboxStats =
(BoundingBoxDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), BoundingBoxDataStatistics.STATS_ID, "aaa");
assertTrue(
(bboxStats.getMinX() == 25)
&& (bboxStats.getMaxX() == 26)
&& (bboxStats.getMinY() == 32)
&& (bboxStats.getMaxY() == 32));
bboxStats =
(BoundingBoxDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), BoundingBoxDataStatistics.STATS_ID, "bbb");
assertTrue(
(bboxStats.getMinX() == 27)
&& (bboxStats.getMaxX() == 27)
&& (bboxStats.getMinY() == 32)
&& (bboxStats.getMaxY() == 32));
bboxStats =
(BoundingBoxDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), BoundingBoxDataStatistics.STATS_ID, "aaa", "bbb");
assertTrue(
(bboxStats.getMinX() == 25)
&& (bboxStats.getMaxX() == 27)
&& (bboxStats.getMinY() == 32)
&& (bboxStats.getMaxY() == 32));
final AtomicBoolean found = new AtomicBoolean(false);
mockDataStore.delete(
new QueryOptions(
adapter,
index,
-1,
new ScanCallback<TestGeometry>() {
@Override
public void entryScanned(
final DataStoreEntryInfo entryInfo, final TestGeometry entry) {
found.getAndSet(true);
}
},
new String[] {"aaa"}),
new DataIdQuery(
adapter.getAdapterId(),
new ByteArrayId("test_pt_2".getBytes(StringUtils.GEOWAVE_CHAR_SET))));
assertFalse(found.get());
try (CloseableIterator<?> it1 =
mockDataStore.query(
new QueryOptions(adapter, index, -1, null, new String[] {"aaa", "bbb"}), query)) {
int count = 0;
while (it1.hasNext()) {
it1.next();
count++;
}
assertEquals(3, count);
}
mockDataStore.delete(
new QueryOptions(adapter, index, -1, null, new String[] {"aaa"}),
new DataIdQuery(
adapter.getAdapterId(),
new ByteArrayId("test_pt".getBytes(StringUtils.GEOWAVE_CHAR_SET))));
try (CloseableIterator<?> it1 =
mockDataStore.query(
new QueryOptions(adapter, index, -1, null, new String[] {"aaa", "bbb"}), query)) {
int count = 0;
while (it1.hasNext()) {
it1.next();
count++;
}
assertEquals(2, count);
}
countStats =
(CountDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), CountDataStatistics.STATS_ID, "aaa");
assertEquals(1, countStats.getCount());
countStats =
(CountDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), CountDataStatistics.STATS_ID, "bbb");
assertEquals(1, countStats.getCount());
bboxStats =
(BoundingBoxDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), BoundingBoxDataStatistics.STATS_ID, "aaa");
assertTrue(
(bboxStats.getMinX() == 25)
&& (bboxStats.getMaxX() == 26)
&& (bboxStats.getMinY() == 32)
&& (bboxStats.getMaxY() == 32));
bboxStats =
(BoundingBoxDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), BoundingBoxDataStatistics.STATS_ID, "bbb");
assertTrue(
(bboxStats.getMinX() == 27)
&& (bboxStats.getMaxX() == 27)
&& (bboxStats.getMinY() == 32)
&& (bboxStats.getMaxY() == 32));
bboxStats =
(BoundingBoxDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), BoundingBoxDataStatistics.STATS_ID, "aaa", "bbb");
assertTrue(
(bboxStats.getMinX() == 25)
&& (bboxStats.getMaxX() == 27)
&& (bboxStats.getMinY() == 32)
&& (bboxStats.getMaxY() == 32));
found.set(false);
assertTrue(
mockDataStore.delete(
new QueryOptions(
adapter,
index,
-1,
new ScanCallback<TestGeometry>() {
@Override
public void entryScanned(
final DataStoreEntryInfo entryInfo, final TestGeometry entry) {
found.getAndSet(true);
}
},
new String[] {"aaa", "bbb"}),
new EverythingQuery()));
try (CloseableIterator<?> it1 =
mockDataStore.query(
new QueryOptions(adapter, index, -1, null, new String[] {"aaa", "bbb"}), query)) {
int count = 0;
while (it1.hasNext()) {
it1.next();
count++;
}
assertEquals(0, count);
}
countStats =
(CountDataStatistics<?>)
statsStore.getDataStatistics(adapter.getAdapterId(), CountDataStatistics.STATS_ID);
assertNull(countStats);
try (IndexWriter indexWriter = mockDataStore.createIndexWriter(index, visWriterBBB)) {
rowId0 =
indexWriter
.write(
adapter,
new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt_2"))
.get(0);
}
countStats =
(CountDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), CountDataStatistics.STATS_ID, "bbb");
assertTrue(countStats != null);
statsStore.deleteObjects(adapter.getAdapterId(), "bbb");
countStats =
(CountDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), CountDataStatistics.STATS_ID, "bbb");
assertNull(countStats);
final RowRangeDataStatistics<?> rowStats =
(RowRangeDataStatistics<?>)
statsStore.getDataStatistics(null, RowRangeDataStatistics.getId(index.getId()), "bbb");
assertTrue(rowStats != null);
}
public void addIndices(final List<PrimaryIndex> indices) {
for (PrimaryIndex index : indices) {
if (!indexCache.indexExists(index.getId())) indexCache.addIndex(index);
}
}
@Test
public void testLocalityGroups() throws IOException {
final PrimaryIndex index = new SpatialDimensionalityTypeProvider().createPrimaryIndex();
final WritableDataAdapter<TestGeometry> adapter = new TestGeometryAdapter();
final String tableName = StringUtils.stringFromBinary(index.getId().getBytes());
final byte[] adapterId = adapter.getAdapterId().getBytes();
accumuloOptions.setUseLocalityGroups(false);
try (IndexWriter indexWriter =
mockDataStore.createIndexWriter(index, DataStoreUtils.DEFAULT_VISIBILITY)) {
final ByteArrayId rowId1 =
indexWriter
.write(
adapter,
new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt_1"))
.get(0);
try {
// as we are not using locality groups, we expect that this will
// return false
assertEquals(false, accumuloOperations.localityGroupExists(tableName, adapterId));
} catch (final AccumuloException | TableNotFoundException e) {
LOGGER.error("Locality Group check failed", e);
}
final TestGeometry geom1 =
(TestGeometry)
mockDataStore
.query(new QueryOptions(), new RowIdQuery(Collections.singletonList(rowId1)))
.next();
// of course, the point is actually stored in this case
assertEquals("test_pt_1", geom1.id);
}
accumuloOptions.setUseLocalityGroups(true);
try (IndexWriter indexWriter =
mockDataStore.createIndexWriter(index, DataStoreUtils.DEFAULT_VISIBILITY)) {
final ByteArrayId rowId2 =
indexWriter
.write(
adapter,
new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt_2"))
.get(0);
try {
// now that locality groups are turned on, we expect this to
// return
// true
assertEquals(true, accumuloOperations.localityGroupExists(tableName, adapterId));
} catch (final AccumuloException | TableNotFoundException e) {
LOGGER.error("Locality Group check failed", e);
}
final TestGeometry geom2 =
(TestGeometry)
mockDataStore
.query(new QueryOptions(), new RowIdQuery(Collections.singletonList(rowId2)))
.next();
// of course, the point is actually stored in this case
assertEquals("test_pt_2", geom2.id);
}
}
@Test
public void testIndexOptions() throws IOException {
final PrimaryIndex index = new SpatialDimensionalityTypeProvider().createPrimaryIndex();
final WritableDataAdapter<TestGeometry> adapter = new TestGeometryAdapter();
accumuloOptions.setCreateTable(false);
accumuloOptions.setPersistIndex(false);
try (IndexWriter indexWriter =
mockDataStore.createIndexWriter(index, DataStoreUtils.DEFAULT_VISIBILITY)) {
final List<ByteArrayId> rowIds =
indexWriter.write(
adapter, new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt"));
// as the table didn't already exist, the flag indicates not to
// create
// it, so no rows will be returned
assertEquals(0, rowIds.size());
assertEquals(false, indexStore.indexExists(index.getId()));
}
accumuloOptions.setCreateTable(true);
try (IndexWriter indexWriter =
mockDataStore.createIndexWriter(index, DataStoreUtils.DEFAULT_VISIBILITY)) {
final ByteArrayId rowId1 =
indexWriter
.write(
adapter,
new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt_1"))
.get(0);
assertFalse(
mockDataStore
.query(new QueryOptions(), new RowIdQuery(Collections.singletonList(rowId1)))
.hasNext());
// as we have chosen not to persist the index, we will not see an
// index
// entry in the index store
assertEquals(false, indexStore.indexExists(index.getId()));
/** Still can query providing the index */
final TestGeometry geom1 =
(TestGeometry)
mockDataStore
.query(
new QueryOptions(adapter, index),
new RowIdQuery(Collections.singletonList(rowId1)))
.next();
// even though we didn't persist the index, the test point was still
// stored
assertEquals("test_pt_1", geom1.id);
}
accumuloOptions.setPersistIndex(true);
try (IndexWriter indexWriter =
mockDataStore.createIndexWriter(index, DataStoreUtils.DEFAULT_VISIBILITY)) {
final ByteArrayId rowId2 =
indexWriter
.write(
adapter,
new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt_2"))
.get(0);
final TestGeometry geom2 =
(TestGeometry)
mockDataStore
.query(new QueryOptions(), new RowIdQuery(Collections.singletonList(rowId2)))
.next();
// as we have chosen to persist the index, we will see the index
// entry
// in the index store
assertEquals(true, indexStore.indexExists(index.getId()));
// of course, the point is actually stored in this case
assertEquals("test_pt_2", geom2.id);
}
}
private static TreeSet<IntermediateSplitInfo> populateIntermediateSplits(
final TreeSet<IntermediateSplitInfo> splits,
final BasicHBaseOperations operations,
final PrimaryIndex index,
final List<DataAdapter<Object>> adapters,
final Map<PrimaryIndex, RowRangeHistogramStatistics<?>> statsCache,
final AdapterStore adapterStore,
final DataStatisticsStore statsStore,
final Integer maxSplits,
final DistributableQuery query,
final String[] authorizations)
throws IOException {
if ((query != null) && !query.isSupported(index)) {
return splits;
}
final HBaseMRRowRange fullrange = getRangeMax(index, adapterStore, statsStore, authorizations);
final String tableName = index.getId().getString();
final NumericIndexStrategy indexStrategy = index.getIndexStrategy();
// Build list of row ranges from query
List<HBaseMRRowRange> ranges = new ArrayList<HBaseMRRowRange>();
final List<ByteArrayRange> constraintRanges;
if (query != null) {
final List<MultiDimensionalNumericData> indexConstraints =
query.getIndexConstraints(indexStrategy);
if ((maxSplits != null) && (maxSplits > 0)) {
constraintRanges =
DataStoreUtils.constraintsToByteArrayRanges(indexConstraints, indexStrategy, maxSplits);
} else {
constraintRanges =
DataStoreUtils.constraintsToByteArrayRanges(indexConstraints, indexStrategy, -1);
}
for (final ByteArrayRange constraintRange : constraintRanges) {
ranges.add(new HBaseMRRowRange(constraintRange));
}
} else {
ranges.add(fullrange);
if (LOGGER.isTraceEnabled()) {
LOGGER.trace("Protected range: " + fullrange);
}
}
final Map<HRegionLocation, Map<HRegionInfo, List<HBaseMRRowRange>>> binnedRanges =
new HashMap<HRegionLocation, Map<HRegionInfo, List<HBaseMRRowRange>>>();
final RegionLocator regionLocator = operations.getRegionLocator(tableName);
while (!ranges.isEmpty()) {
ranges = binRanges(ranges, binnedRanges, regionLocator);
}
for (final Entry<HRegionLocation, Map<HRegionInfo, List<HBaseMRRowRange>>> locationEntry :
binnedRanges.entrySet()) {
final String hostname = locationEntry.getKey().getHostname();
for (final Entry<HRegionInfo, List<HBaseMRRowRange>> regionEntry :
locationEntry.getValue().entrySet()) {
final Map<PrimaryIndex, List<RangeLocationPair>> splitInfo =
new HashMap<PrimaryIndex, List<RangeLocationPair>>();
final List<RangeLocationPair> rangeList = new ArrayList<RangeLocationPair>();
for (final HBaseMRRowRange range : regionEntry.getValue()) {
final double cardinality =
getCardinality(
getHistStats(
index, adapters, adapterStore, statsStore, statsCache, authorizations),
range);
if (range.intersects(fullrange)) {
rangeList.add(
new RangeLocationPair(range, hostname, cardinality < 1 ? 1.0 : cardinality));
} else {
LOGGER.info("Query split outside of range");
}
if (LOGGER.isTraceEnabled()) {
LOGGER.warn("Clipped range: " + rangeList.get(rangeList.size() - 1).getRange());
}
}
if (!rangeList.isEmpty()) {
splitInfo.put(index, rangeList);
splits.add(new IntermediateSplitInfo(splitInfo));
}
}
}
return splits;
}
/** Initialize a scanner over the given input split using this task attempt configuration. */
@Override
public void initialize(final InputSplit inSplit, final TaskAttemptContext attempt)
throws IOException {
split = (GeoWaveAccumuloInputSplit) inSplit;
numKeysRead = 0;
final Map<RangeLocationPair, CloseableIterator<?>> iteratorsPerRange =
new LinkedHashMap<RangeLocationPair, CloseableIterator<?>>();
final Set<PrimaryIndex> indices = split.getIndices();
BigDecimal sum = BigDecimal.ZERO;
final Map<RangeLocationPair, BigDecimal> incrementalRangeSums =
new LinkedHashMap<RangeLocationPair, BigDecimal>();
for (final PrimaryIndex i : indices) {
final List<RangeLocationPair> ranges = split.getRanges(i);
List<QueryFilter> queryFilters = null;
if (query != null) {
queryFilters = query.createFilters(i.getIndexModel());
}
for (final RangeLocationPair r : ranges) {
final QueryOptions rangeQueryOptions = new QueryOptions(queryOptions);
rangeQueryOptions.setIndex(i);
iteratorsPerRange.put(
r,
new InputFormatAccumuloRangeQuery(
adapterStore,
i,
r.getRange(),
queryFilters,
isOutputWritable,
rangeQueryOptions)
.query(
accumuloOperations,
adapterStore,
rangeQueryOptions.getMaxResolutionSubsamplingPerDimension(),
rangeQueryOptions.getLimit()));
incrementalRangeSums.put(r, sum);
sum = sum.add(BigDecimal.valueOf(r.getCardinality()));
}
}
// finally we can compute percent progress
progressPerRange = new LinkedHashMap<RangeLocationPair, ProgressPerRange>();
RangeLocationPair prevRangeIndex = null;
float prevProgress = 0f;
for (final Entry<RangeLocationPair, BigDecimal> entry : incrementalRangeSums.entrySet()) {
final BigDecimal value = entry.getValue();
final float progress = value.divide(sum, RoundingMode.HALF_UP).floatValue();
if (prevRangeIndex != null) {
progressPerRange.put(prevRangeIndex, new ProgressPerRange(prevProgress, progress));
}
prevRangeIndex = entry.getKey();
prevProgress = progress;
}
progressPerRange.put(prevRangeIndex, new ProgressPerRange(prevProgress, 1f));
// concatenate iterators
iterator =
new CloseableIteratorWrapper<Object>(
new Closeable() {
@Override
public void close() throws IOException {
for (final CloseableIterator<?> it : iteratorsPerRange.values()) {
it.close();
}
}
},
concatenateWithCallback(
iteratorsPerRange.entrySet().iterator(),
new NextRangeCallback() {
@Override
public void setRange(final RangeLocationPair indexPair) {
currentGeoWaveRangeIndexPair = indexPair;
}
}));
}
