@Override
public boolean deleteObjects(
final ByteArrayId primaryId, final ByteArrayId secondaryId, final String... authorizations) {
if (primaryId != null) {
return accumuloOperations.delete(
getTablename(),
primaryId,
getColumnFamily(),
getColumnQualifier(secondaryId),
authorizations);
}
try {
final BatchScanner scanner = getScanner(null, secondaryId, authorizations);
final Iterator<Entry<Key, Value>> it = scanner.iterator();
try (final CloseableIterator<?> cit =
new CloseableIteratorWrapper<T>(
new ScannerClosableWrapper(scanner), new DeleteIteratorWrapper(it, authorizations))) {
while (cit.hasNext()) {
deleteObjectFromCache(getPrimaryId((T) cit.next()), secondaryId);
}
} catch (final IOException e) {
LOGGER.error("Unable to delete objects", e);
}
} catch (final TableNotFoundException e) {
LOGGER.error("Unable to find objects, table '" + getTablename() + "' does not exist", e);
}
return true;
}
private void queryNoDataMergeStrategy(final String coverageName, final int tileSize)
throws AccumuloException, AccumuloSecurityException, IOException {
final DataStore dataStore = dataStoreOptions.createDataStore();
try (CloseableIterator<?> it =
dataStore.query(
new QueryOptions(new ByteArrayId(coverageName), null), new EverythingQuery())) {
// the expected outcome is:
// band 1,2,3,4,5,6 has every value set correctly, band 0 has every
// even row set correctly and every odd row should be NaN, and band
// 7 has the upper quadrant as NaN and the rest set
final GridCoverage coverage = (GridCoverage) it.next();
final Raster raster = coverage.getRenderedImage().getData();
Assert.assertEquals(tileSize, raster.getWidth());
Assert.assertEquals(tileSize, raster.getHeight());
for (int x = 0; x < tileSize; x++) {
for (int y = 0; y < tileSize; y++) {
for (int b = 1; b < 7; b++) {
Assert.assertEquals(
"x=" + x + ",y=" + y + ",b=" + b,
getValue(x, y, b, tileSize),
raster.getSampleDouble(x, y, b));
}
if ((y % 2) == 0) {
Assert.assertEquals(
"x=" + x + ",y=" + y + ",b=0",
getValue(x, y, 0, tileSize),
raster.getSampleDouble(x, y, 0));
} else {
Assert.assertEquals(
"x=" + x + ",y=" + y + ",b=0", Double.NaN, raster.getSampleDouble(x, y, 0));
}
if ((x > ((tileSize * 3) / 4)) && (y > ((tileSize * 3) / 4))) {
Assert.assertEquals(
"x=" + x + ",y=" + y + ",b=7", Double.NaN, raster.getSampleDouble(x, y, 7));
} else {
Assert.assertEquals(
"x=" + x + ",y=" + y + ",b=7",
getValue(x, y, 7, tileSize),
raster.getSampleDouble(x, y, 7));
}
}
}
// there should be exactly one
Assert.assertFalse(it.hasNext());
}
}
private static void executeBoundingBoxQuery() throws IOException {
System.out.println(
"Constructing bounding box for the area contained by [Baltimore, MD and Richmond, VA.");
final Geometry boundingBox =
GeometryUtils.GEOMETRY_FACTORY.toGeometry(new Envelope(baltimore, richmond));
System.out.println("Executing query, expecting to match ALL points...");
try (final CloseableIterator<SimpleFeature> iterator =
dataStore.query(new QueryOptions(index), new SpatialQuery(boundingBox))) {
while (iterator.hasNext()) {
System.out.println("Query match: " + iterator.next().getID());
}
}
}
private void queryGeneralPurpose(
final String coverageName,
final int tileSize,
final double westLon,
final double eastLon,
final double southLat,
final double northLat,
final int numBands,
final int numRasters,
final ExpectedValue expectedValue)
throws AccumuloException, AccumuloSecurityException, IOException {
final DataStore dataStore = dataStoreOptions.createDataStore();
try (CloseableIterator<?> it =
dataStore.query(
new QueryOptions(new ByteArrayId(coverageName), null),
new IndexOnlySpatialQuery(
new GeometryFactory()
.toGeometry(new Envelope(westLon, eastLon, southLat, northLat))))) {
// the expected outcome is:
// band 1,2,3,4,5,6 has every value set correctly, band 0 has every
// even row set correctly and every odd row should be NaN, and band
// 7 has the upper quadrant as NaN and the rest set
final GridCoverage coverage = (GridCoverage) it.next();
final Raster raster = coverage.getRenderedImage().getData();
Assert.assertEquals(tileSize, raster.getWidth());
Assert.assertEquals(tileSize, raster.getHeight());
for (int x = 0; x < tileSize; x++) {
for (int y = 0; y < tileSize; y++) {
for (int b = 0; b < numBands; b++) {
Assert.assertEquals(
"x=" + x + ",y=" + y + ",b=" + b,
expectedValue.getExpectedValue(x, y, b, numRasters, tileSize),
raster.getSampleDouble(x, y, b),
DOUBLE_TOLERANCE);
}
}
}
// there should be exactly one
Assert.assertFalse(it.hasNext());
}
}
@Override
public void close() {
if (iterator != null) {
try {
iterator.close();
} catch (final IOException e) {
LOGGER.warn("Unable to close iterator", e);
}
}
}
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
if (iterator != null) {
if (iterator.hasNext()) {
++numKeysRead;
final Object value = iterator.next();
if (value instanceof Entry) {
final Entry<GeoWaveInputKey, T> entry = (Entry<GeoWaveInputKey, T>) value;
currentGeoWaveKey = entry.getKey();
if (currentGeoWaveKey == null) {
currentAccumuloKey = null;
} else {
currentAccumuloKey = currentGeoWaveKey.getAccumuloKey();
}
currentValue = entry.getValue();
}
return true;
}
}
return false;
}
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
if (iterator != null) {
if (iterator.hasNext()) {
++numKeysRead;
final Object value = iterator.next();
if (value instanceof Entry) {
final Entry<GeoWaveInputKey, T> entry = (Entry<GeoWaveInputKey, T>) value;
currentGeoWaveKey = entry.getKey();
if (currentGeoWaveKey == null) {
currentAccumuloKey = null;
} else if (currentGeoWaveKey.getInsertionId() != null) {
// just use the insertion ID for progress
currentAccumuloKey = new Key(new Text(currentGeoWaveKey.getInsertionId().getBytes()));
}
currentValue = entry.getValue();
}
return true;
}
}
return false;
}
private static void executePolygonQuery() throws IOException {
System.out.println(
"Constructing polygon for the area contained by [Baltimore, MD; Richmond, VA; Harrisonburg, VA].");
final Polygon polygon =
GeometryUtils.GEOMETRY_FACTORY.createPolygon(
new Coordinate[] {baltimore, richmond, harrisonburg, baltimore});
System.out.println("Executing query, expecting to match ALL points...");
/**
* NOTICE: In this query, the adapter is added to the query options. If an index has data from
* more than one adapter, the data associated with a specific adapter can be selected.
*/
try (final CloseableIterator<SimpleFeature> closableIterator =
dataStore.query(new QueryOptions(ADAPTER, index), new SpatialQuery(polygon))) {
while (closableIterator.hasNext()) {
System.out.println("Query match: " + closableIterator.next().getID());
}
}
}
@Override
protected long runQuery(
final GeotoolsFeatureDataAdapter adapter,
final ByteArrayId adapterId,
final ByteArrayId indexId,
final DataStore dataStore,
final boolean debug) {
long count = 0;
try (final CloseableIterator<Object> it =
dataStore.query(new QueryOptions(adapterId, indexId), null)) {
while (it.hasNext()) {
if (debug) {
System.out.println(it.next());
} else {
it.next();
}
count++;
}
} catch (final IOException e) {
e.printStackTrace();
}
return count;
}
private void runtest() throws IOException {
final PrimaryIndex index = new SpatialDimensionalityTypeProvider().createPrimaryIndex();
final WritableDataAdapter<TestGeometry> adapter = new TestGeometryAdapter();
final Geometry testGeoFilter =
factory.createPolygon(
new Coordinate[] {
new Coordinate(24, 33),
new Coordinate(28, 33),
new Coordinate(28, 31),
new Coordinate(24, 31),
new Coordinate(24, 33)
});
ByteArrayId rowId0, rowId1;
try (IndexWriter indexWriter =
mockDataStore.createIndexWriter(index, DataStoreUtils.DEFAULT_VISIBILITY)) {
rowId0 =
indexWriter
.write(
adapter,
new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt"),
visWriterAAA)
.get(0);
rowId1 =
indexWriter
.write(
adapter,
new TestGeometry(factory.createPoint(new Coordinate(26, 32)), "test_pt_1"),
visWriterAAA)
.get(0);
indexWriter
.write(
adapter,
new TestGeometry(factory.createPoint(new Coordinate(27, 32)), "test_pt_2"),
visWriterBBB)
.get(0);
}
final SpatialQuery query = new SpatialQuery(testGeoFilter);
try (CloseableIterator<?> it1 =
mockDataStore.query(
new QueryOptions(adapter, index, -1, null, new String[] {"aaa", "bbb"}), query)) {
int count = 0;
while (it1.hasNext()) {
it1.next();
count++;
}
assertEquals(3, count);
}
CountDataStatistics<?> countStats =
(CountDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), CountDataStatistics.STATS_ID, "aaa", "bbb");
assertEquals(3, countStats.getCount());
countStats =
(CountDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), CountDataStatistics.STATS_ID, "aaa");
assertEquals(2, countStats.getCount());
countStats =
(CountDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), CountDataStatistics.STATS_ID, "bbb");
assertEquals(1, countStats.getCount());
BoundingBoxDataStatistics<?> bboxStats =
(BoundingBoxDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), BoundingBoxDataStatistics.STATS_ID, "aaa");
assertTrue(
(bboxStats.getMinX() == 25)
&& (bboxStats.getMaxX() == 26)
&& (bboxStats.getMinY() == 32)
&& (bboxStats.getMaxY() == 32));
bboxStats =
(BoundingBoxDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), BoundingBoxDataStatistics.STATS_ID, "bbb");
assertTrue(
(bboxStats.getMinX() == 27)
&& (bboxStats.getMaxX() == 27)
&& (bboxStats.getMinY() == 32)
&& (bboxStats.getMaxY() == 32));
bboxStats =
(BoundingBoxDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), BoundingBoxDataStatistics.STATS_ID, "aaa", "bbb");
assertTrue(
(bboxStats.getMinX() == 25)
&& (bboxStats.getMaxX() == 27)
&& (bboxStats.getMinY() == 32)
&& (bboxStats.getMaxY() == 32));
final AtomicBoolean found = new AtomicBoolean(false);
mockDataStore.delete(
new QueryOptions(
adapter,
index,
-1,
new ScanCallback<TestGeometry>() {
@Override
public void entryScanned(
final DataStoreEntryInfo entryInfo, final TestGeometry entry) {
found.getAndSet(true);
}
},
new String[] {"aaa"}),
new DataIdQuery(
adapter.getAdapterId(),
new ByteArrayId("test_pt_2".getBytes(StringUtils.GEOWAVE_CHAR_SET))));
assertFalse(found.get());
try (CloseableIterator<?> it1 =
mockDataStore.query(
new QueryOptions(adapter, index, -1, null, new String[] {"aaa", "bbb"}), query)) {
int count = 0;
while (it1.hasNext()) {
it1.next();
count++;
}
assertEquals(3, count);
}
mockDataStore.delete(
new QueryOptions(adapter, index, -1, null, new String[] {"aaa"}),
new DataIdQuery(
adapter.getAdapterId(),
new ByteArrayId("test_pt".getBytes(StringUtils.GEOWAVE_CHAR_SET))));
try (CloseableIterator<?> it1 =
mockDataStore.query(
new QueryOptions(adapter, index, -1, null, new String[] {"aaa", "bbb"}), query)) {
int count = 0;
while (it1.hasNext()) {
it1.next();
count++;
}
assertEquals(2, count);
}
countStats =
(CountDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), CountDataStatistics.STATS_ID, "aaa");
assertEquals(1, countStats.getCount());
countStats =
(CountDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), CountDataStatistics.STATS_ID, "bbb");
assertEquals(1, countStats.getCount());
bboxStats =
(BoundingBoxDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), BoundingBoxDataStatistics.STATS_ID, "aaa");
assertTrue(
(bboxStats.getMinX() == 25)
&& (bboxStats.getMaxX() == 26)
&& (bboxStats.getMinY() == 32)
&& (bboxStats.getMaxY() == 32));
bboxStats =
(BoundingBoxDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), BoundingBoxDataStatistics.STATS_ID, "bbb");
assertTrue(
(bboxStats.getMinX() == 27)
&& (bboxStats.getMaxX() == 27)
&& (bboxStats.getMinY() == 32)
&& (bboxStats.getMaxY() == 32));
bboxStats =
(BoundingBoxDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), BoundingBoxDataStatistics.STATS_ID, "aaa", "bbb");
assertTrue(
(bboxStats.getMinX() == 25)
&& (bboxStats.getMaxX() == 27)
&& (bboxStats.getMinY() == 32)
&& (bboxStats.getMaxY() == 32));
found.set(false);
assertTrue(
mockDataStore.delete(
new QueryOptions(
adapter,
index,
-1,
new ScanCallback<TestGeometry>() {
@Override
public void entryScanned(
final DataStoreEntryInfo entryInfo, final TestGeometry entry) {
found.getAndSet(true);
}
},
new String[] {"aaa", "bbb"}),
new EverythingQuery()));
try (CloseableIterator<?> it1 =
mockDataStore.query(
new QueryOptions(adapter, index, -1, null, new String[] {"aaa", "bbb"}), query)) {
int count = 0;
while (it1.hasNext()) {
it1.next();
count++;
}
assertEquals(0, count);
}
countStats =
(CountDataStatistics<?>)
statsStore.getDataStatistics(adapter.getAdapterId(), CountDataStatistics.STATS_ID);
assertNull(countStats);
try (IndexWriter indexWriter = mockDataStore.createIndexWriter(index, visWriterBBB)) {
rowId0 =
indexWriter
.write(
adapter,
new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt_2"))
.get(0);
}
countStats =
(CountDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), CountDataStatistics.STATS_ID, "bbb");
assertTrue(countStats != null);
statsStore.deleteObjects(adapter.getAdapterId(), "bbb");
countStats =
(CountDataStatistics<?>)
statsStore.getDataStatistics(
adapter.getAdapterId(), CountDataStatistics.STATS_ID, "bbb");
assertNull(countStats);
final RowRangeDataStatistics<?> rowStats =
(RowRangeDataStatistics<?>)
statsStore.getDataStatistics(null, RowRangeDataStatistics.getId(index.getId()), "bbb");
assertTrue(rowStats != null);
}
@Test
public void testDeleteAll() throws IOException {
final PrimaryIndex index = new SpatialDimensionalityTypeProvider().createPrimaryIndex();
final WritableDataAdapter<TestGeometry> adapter0 = new TestGeometryAdapter();
final WritableDataAdapter<TestGeometry> adapter1 = new AnotherAdapter();
accumuloOptions.setUseAltIndex(true);
try (IndexWriter indexWriter =
mockDataStore.createIndexWriter(index, DataStoreUtils.DEFAULT_VISIBILITY)) {
final ByteArrayId rowId0 =
indexWriter
.write(
adapter0,
new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt_0"))
.get(0);
final ByteArrayId rowId1 =
indexWriter
.write(
adapter1,
new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt_1"))
.get(0);
}
CloseableIterator it =
mockDataStore.query(new QueryOptions(adapter0, index), new EverythingQuery());
int count = 0;
while (it.hasNext()) {
it.next();
count++;
}
assertEquals(1, count);
it = mockDataStore.query(new QueryOptions(adapter1, index), new EverythingQuery());
count = 0;
while (it.hasNext()) {
it.next();
count++;
}
assertEquals(1, count);
it = mockDataStore.query(new QueryOptions(index), new EverythingQuery());
count = 0;
while (it.hasNext()) {
it.next();
count++;
}
assertEquals(2, count);
// delete entry by data id & adapter id
assertTrue(mockDataStore.delete(new QueryOptions(adapter0, index), new EverythingQuery()));
it = mockDataStore.query(new QueryOptions(index), new EverythingQuery());
count = 0;
while (it.hasNext()) {
it.next();
count++;
}
assertEquals(1, count);
it = mockDataStore.query(new QueryOptions(adapter0, index), new EverythingQuery());
count = 0;
while (it.hasNext()) {
it.next();
count++;
}
assertEquals(0, count);
try (IndexWriter indexWriter =
mockDataStore.createIndexWriter(index, DataStoreUtils.DEFAULT_VISIBILITY)) {
final ByteArrayId rowId0 =
indexWriter
.write(
adapter0,
new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt_2"))
.get(0);
}
it = mockDataStore.query(new QueryOptions(index), new EverythingQuery());
count = 0;
while (it.hasNext()) {
it.next();
count++;
}
assertEquals(2, count);
assertTrue(mockDataStore.delete(new QueryOptions(), new EverythingQuery()));
it = mockDataStore.query(new QueryOptions(index), new EverythingQuery());
count = 0;
while (it.hasNext()) {
it.next();
count++;
}
assertEquals(0, count);
}
@Test
public void testAdapterOptions() throws IOException {
final PrimaryIndex index = new SpatialDimensionalityTypeProvider().createPrimaryIndex();
final WritableDataAdapter<TestGeometry> adapter = new TestGeometryAdapter();
accumuloOptions.setPersistAdapter(false);
try (IndexWriter indexWriter =
mockDataStore.createIndexWriter(index, DataStoreUtils.DEFAULT_VISIBILITY)) {
final ByteArrayId rowId1 =
indexWriter
.write(
adapter,
new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt_1"))
.get(0);
assertFalse(
mockDataStore
.query(new QueryOptions(), new RowIdQuery(Collections.singletonList(rowId1)))
.hasNext());
}
try (IndexWriter indexWriter =
mockDataStore.createIndexWriter(index, DataStoreUtils.DEFAULT_VISIBILITY)) {
final ByteArrayId rowId1 =
indexWriter
.write(
adapter,
new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt_1"))
.get(0);
assertFalse(
mockDataStore
.query(new QueryOptions(), new RowIdQuery(Collections.singletonList(rowId1)))
.hasNext());
try (final CloseableIterator<TestGeometry> geomItr =
mockDataStore.query(new QueryOptions(adapter, index), new EverythingQuery())) {
final TestGeometry geom1 = geomItr.next();
// specifying the adapter, this method returns the entry
assertEquals("test_pt_1", geom1.id);
}
// the adapter should not exist in the metadata table
assertEquals(false, adapterStore.adapterExists(adapter.getAdapterId()));
}
accumuloOptions.setPersistAdapter(true);
try (IndexWriter indexWriter =
mockDataStore.createIndexWriter(index, DataStoreUtils.DEFAULT_VISIBILITY)) {
final ByteArrayId rowId2 =
indexWriter
.write(
adapter,
new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt_2"))
.get(0);
try (final CloseableIterator<?> geomItr =
mockDataStore.query(
new QueryOptions(), new RowIdQuery(Collections.singletonList(rowId2)))) {
assertTrue(geomItr.hasNext());
final TestGeometry geom2 = (TestGeometry) geomItr.next();
// specifying the adapter, this method returns the entry
assertEquals("test_pt_2", geom2.id);
}
try (final CloseableIterator<TestGeometry> geomItr =
mockDataStore.query(new QueryOptions(adapter, index), null)) {
while (geomItr.hasNext()) {
final TestGeometry geom2 = geomItr.next();
// specifying the adapter, this method returns the entry
assertTrue(Arrays.asList("test_pt_2", "test_pt_1").contains(geom2.id));
}
}
// the adapter should not exist in the metadata table
assertEquals(true, adapterStore.adapterExists(adapter.getAdapterId()));
}
// the adapter should exist in the metadata table
assertEquals(true, adapterStore.adapterExists(adapter.getAdapterId()));
}