Esempio n. 1
0
  private static RowRangeHistogramStatistics<?> getRangeStats(
      final PrimaryIndex index,
      final List<DataAdapter<Object>> adapters,
      final AdapterStore adapterStore,
      final DataStatisticsStore store,
      final String[] authorizations) {
    RowRangeHistogramStatistics<?> singleStats = null;
    for (final DataAdapter<?> adapter : adapters) {
      final RowRangeHistogramStatistics<?> rowStat =
          (RowRangeHistogramStatistics<?>)
              store.getDataStatistics(
                  adapter.getAdapterId(),
                  RowRangeHistogramStatistics.composeId(index.getId()),
                  authorizations);
      if (singleStats == null) {
        singleStats = rowStat;
      } else {
        singleStats.merge(rowStat);
      }
    }

    return singleStats;
  }
  private void runtest() throws IOException {

    final PrimaryIndex index = new SpatialDimensionalityTypeProvider().createPrimaryIndex();
    final WritableDataAdapter<TestGeometry> adapter = new TestGeometryAdapter();

    final Geometry testGeoFilter =
        factory.createPolygon(
            new Coordinate[] {
              new Coordinate(24, 33),
              new Coordinate(28, 33),
              new Coordinate(28, 31),
              new Coordinate(24, 31),
              new Coordinate(24, 33)
            });

    ByteArrayId rowId0, rowId1;
    try (IndexWriter indexWriter =
        mockDataStore.createIndexWriter(index, DataStoreUtils.DEFAULT_VISIBILITY)) {
      rowId0 =
          indexWriter
              .write(
                  adapter,
                  new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt"),
                  visWriterAAA)
              .get(0);
      rowId1 =
          indexWriter
              .write(
                  adapter,
                  new TestGeometry(factory.createPoint(new Coordinate(26, 32)), "test_pt_1"),
                  visWriterAAA)
              .get(0);
      indexWriter
          .write(
              adapter,
              new TestGeometry(factory.createPoint(new Coordinate(27, 32)), "test_pt_2"),
              visWriterBBB)
          .get(0);
    }

    final SpatialQuery query = new SpatialQuery(testGeoFilter);

    try (CloseableIterator<?> it1 =
        mockDataStore.query(
            new QueryOptions(adapter, index, -1, null, new String[] {"aaa", "bbb"}), query)) {
      int count = 0;
      while (it1.hasNext()) {
        it1.next();
        count++;
      }
      assertEquals(3, count);
    }

    CountDataStatistics<?> countStats =
        (CountDataStatistics<?>)
            statsStore.getDataStatistics(
                adapter.getAdapterId(), CountDataStatistics.STATS_ID, "aaa", "bbb");
    assertEquals(3, countStats.getCount());

    countStats =
        (CountDataStatistics<?>)
            statsStore.getDataStatistics(
                adapter.getAdapterId(), CountDataStatistics.STATS_ID, "aaa");
    assertEquals(2, countStats.getCount());

    countStats =
        (CountDataStatistics<?>)
            statsStore.getDataStatistics(
                adapter.getAdapterId(), CountDataStatistics.STATS_ID, "bbb");
    assertEquals(1, countStats.getCount());

    BoundingBoxDataStatistics<?> bboxStats =
        (BoundingBoxDataStatistics<?>)
            statsStore.getDataStatistics(
                adapter.getAdapterId(), BoundingBoxDataStatistics.STATS_ID, "aaa");
    assertTrue(
        (bboxStats.getMinX() == 25)
            && (bboxStats.getMaxX() == 26)
            && (bboxStats.getMinY() == 32)
            && (bboxStats.getMaxY() == 32));

    bboxStats =
        (BoundingBoxDataStatistics<?>)
            statsStore.getDataStatistics(
                adapter.getAdapterId(), BoundingBoxDataStatistics.STATS_ID, "bbb");
    assertTrue(
        (bboxStats.getMinX() == 27)
            && (bboxStats.getMaxX() == 27)
            && (bboxStats.getMinY() == 32)
            && (bboxStats.getMaxY() == 32));

    bboxStats =
        (BoundingBoxDataStatistics<?>)
            statsStore.getDataStatistics(
                adapter.getAdapterId(), BoundingBoxDataStatistics.STATS_ID, "aaa", "bbb");
    assertTrue(
        (bboxStats.getMinX() == 25)
            && (bboxStats.getMaxX() == 27)
            && (bboxStats.getMinY() == 32)
            && (bboxStats.getMaxY() == 32));

    final AtomicBoolean found = new AtomicBoolean(false);
    mockDataStore.delete(
        new QueryOptions(
            adapter,
            index,
            -1,
            new ScanCallback<TestGeometry>() {

              @Override
              public void entryScanned(
                  final DataStoreEntryInfo entryInfo, final TestGeometry entry) {
                found.getAndSet(true);
              }
            },
            new String[] {"aaa"}),
        new DataIdQuery(
            adapter.getAdapterId(),
            new ByteArrayId("test_pt_2".getBytes(StringUtils.GEOWAVE_CHAR_SET))));
    assertFalse(found.get());

    try (CloseableIterator<?> it1 =
        mockDataStore.query(
            new QueryOptions(adapter, index, -1, null, new String[] {"aaa", "bbb"}), query)) {
      int count = 0;
      while (it1.hasNext()) {
        it1.next();
        count++;
      }
      assertEquals(3, count);
    }

    mockDataStore.delete(
        new QueryOptions(adapter, index, -1, null, new String[] {"aaa"}),
        new DataIdQuery(
            adapter.getAdapterId(),
            new ByteArrayId("test_pt".getBytes(StringUtils.GEOWAVE_CHAR_SET))));

    try (CloseableIterator<?> it1 =
        mockDataStore.query(
            new QueryOptions(adapter, index, -1, null, new String[] {"aaa", "bbb"}), query)) {
      int count = 0;
      while (it1.hasNext()) {
        it1.next();
        count++;
      }
      assertEquals(2, count);
    }

    countStats =
        (CountDataStatistics<?>)
            statsStore.getDataStatistics(
                adapter.getAdapterId(), CountDataStatistics.STATS_ID, "aaa");
    assertEquals(1, countStats.getCount());

    countStats =
        (CountDataStatistics<?>)
            statsStore.getDataStatistics(
                adapter.getAdapterId(), CountDataStatistics.STATS_ID, "bbb");
    assertEquals(1, countStats.getCount());

    bboxStats =
        (BoundingBoxDataStatistics<?>)
            statsStore.getDataStatistics(
                adapter.getAdapterId(), BoundingBoxDataStatistics.STATS_ID, "aaa");
    assertTrue(
        (bboxStats.getMinX() == 25)
            && (bboxStats.getMaxX() == 26)
            && (bboxStats.getMinY() == 32)
            && (bboxStats.getMaxY() == 32));

    bboxStats =
        (BoundingBoxDataStatistics<?>)
            statsStore.getDataStatistics(
                adapter.getAdapterId(), BoundingBoxDataStatistics.STATS_ID, "bbb");
    assertTrue(
        (bboxStats.getMinX() == 27)
            && (bboxStats.getMaxX() == 27)
            && (bboxStats.getMinY() == 32)
            && (bboxStats.getMaxY() == 32));

    bboxStats =
        (BoundingBoxDataStatistics<?>)
            statsStore.getDataStatistics(
                adapter.getAdapterId(), BoundingBoxDataStatistics.STATS_ID, "aaa", "bbb");
    assertTrue(
        (bboxStats.getMinX() == 25)
            && (bboxStats.getMaxX() == 27)
            && (bboxStats.getMinY() == 32)
            && (bboxStats.getMaxY() == 32));

    found.set(false);

    assertTrue(
        mockDataStore.delete(
            new QueryOptions(
                adapter,
                index,
                -1,
                new ScanCallback<TestGeometry>() {

                  @Override
                  public void entryScanned(
                      final DataStoreEntryInfo entryInfo, final TestGeometry entry) {
                    found.getAndSet(true);
                  }
                },
                new String[] {"aaa", "bbb"}),
            new EverythingQuery()));

    try (CloseableIterator<?> it1 =
        mockDataStore.query(
            new QueryOptions(adapter, index, -1, null, new String[] {"aaa", "bbb"}), query)) {
      int count = 0;
      while (it1.hasNext()) {
        it1.next();
        count++;
      }
      assertEquals(0, count);
    }

    countStats =
        (CountDataStatistics<?>)
            statsStore.getDataStatistics(adapter.getAdapterId(), CountDataStatistics.STATS_ID);
    assertNull(countStats);

    try (IndexWriter indexWriter = mockDataStore.createIndexWriter(index, visWriterBBB)) {
      rowId0 =
          indexWriter
              .write(
                  adapter,
                  new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt_2"))
              .get(0);
    }

    countStats =
        (CountDataStatistics<?>)
            statsStore.getDataStatistics(
                adapter.getAdapterId(), CountDataStatistics.STATS_ID, "bbb");
    assertTrue(countStats != null);

    statsStore.deleteObjects(adapter.getAdapterId(), "bbb");

    countStats =
        (CountDataStatistics<?>)
            statsStore.getDataStatistics(
                adapter.getAdapterId(), CountDataStatistics.STATS_ID, "bbb");
    assertNull(countStats);

    final RowRangeDataStatistics<?> rowStats =
        (RowRangeDataStatistics<?>)
            statsStore.getDataStatistics(null, RowRangeDataStatistics.getId(index.getId()), "bbb");

    assertTrue(rowStats != null);
  }
Esempio n. 3
0
 public void addIndices(final List<PrimaryIndex> indices) {
   for (PrimaryIndex index : indices) {
     if (!indexCache.indexExists(index.getId())) indexCache.addIndex(index);
   }
 }
  @Test
  public void testLocalityGroups() throws IOException {

    final PrimaryIndex index = new SpatialDimensionalityTypeProvider().createPrimaryIndex();
    final WritableDataAdapter<TestGeometry> adapter = new TestGeometryAdapter();

    final String tableName = StringUtils.stringFromBinary(index.getId().getBytes());
    final byte[] adapterId = adapter.getAdapterId().getBytes();

    accumuloOptions.setUseLocalityGroups(false);

    try (IndexWriter indexWriter =
        mockDataStore.createIndexWriter(index, DataStoreUtils.DEFAULT_VISIBILITY)) {
      final ByteArrayId rowId1 =
          indexWriter
              .write(
                  adapter,
                  new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt_1"))
              .get(0);

      try {
        // as we are not using locality groups, we expect that this will
        // return false
        assertEquals(false, accumuloOperations.localityGroupExists(tableName, adapterId));
      } catch (final AccumuloException | TableNotFoundException e) {
        LOGGER.error("Locality Group check failed", e);
      }

      final TestGeometry geom1 =
          (TestGeometry)
              mockDataStore
                  .query(new QueryOptions(), new RowIdQuery(Collections.singletonList(rowId1)))
                  .next();

      // of course, the point is actually stored in this case
      assertEquals("test_pt_1", geom1.id);
    }

    accumuloOptions.setUseLocalityGroups(true);

    try (IndexWriter indexWriter =
        mockDataStore.createIndexWriter(index, DataStoreUtils.DEFAULT_VISIBILITY)) {
      final ByteArrayId rowId2 =
          indexWriter
              .write(
                  adapter,
                  new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt_2"))
              .get(0);

      try {
        // now that locality groups are turned on, we expect this to
        // return
        // true
        assertEquals(true, accumuloOperations.localityGroupExists(tableName, adapterId));
      } catch (final AccumuloException | TableNotFoundException e) {
        LOGGER.error("Locality Group check failed", e);
      }
      final TestGeometry geom2 =
          (TestGeometry)
              mockDataStore
                  .query(new QueryOptions(), new RowIdQuery(Collections.singletonList(rowId2)))
                  .next();

      // of course, the point is actually stored in this case
      assertEquals("test_pt_2", geom2.id);
    }
  }
  @Test
  public void testIndexOptions() throws IOException {

    final PrimaryIndex index = new SpatialDimensionalityTypeProvider().createPrimaryIndex();
    final WritableDataAdapter<TestGeometry> adapter = new TestGeometryAdapter();

    accumuloOptions.setCreateTable(false);
    accumuloOptions.setPersistIndex(false);

    try (IndexWriter indexWriter =
        mockDataStore.createIndexWriter(index, DataStoreUtils.DEFAULT_VISIBILITY)) {
      final List<ByteArrayId> rowIds =
          indexWriter.write(
              adapter, new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt"));

      // as the table didn't already exist, the flag indicates not to
      // create
      // it, so no rows will be returned
      assertEquals(0, rowIds.size());

      assertEquals(false, indexStore.indexExists(index.getId()));
    }

    accumuloOptions.setCreateTable(true);

    try (IndexWriter indexWriter =
        mockDataStore.createIndexWriter(index, DataStoreUtils.DEFAULT_VISIBILITY)) {
      final ByteArrayId rowId1 =
          indexWriter
              .write(
                  adapter,
                  new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt_1"))
              .get(0);

      assertFalse(
          mockDataStore
              .query(new QueryOptions(), new RowIdQuery(Collections.singletonList(rowId1)))
              .hasNext());

      // as we have chosen not to persist the index, we will not see an
      // index
      // entry in the index store
      assertEquals(false, indexStore.indexExists(index.getId()));

      /** Still can query providing the index */
      final TestGeometry geom1 =
          (TestGeometry)
              mockDataStore
                  .query(
                      new QueryOptions(adapter, index),
                      new RowIdQuery(Collections.singletonList(rowId1)))
                  .next();

      // even though we didn't persist the index, the test point was still
      // stored
      assertEquals("test_pt_1", geom1.id);
    }

    accumuloOptions.setPersistIndex(true);

    try (IndexWriter indexWriter =
        mockDataStore.createIndexWriter(index, DataStoreUtils.DEFAULT_VISIBILITY)) {
      final ByteArrayId rowId2 =
          indexWriter
              .write(
                  adapter,
                  new TestGeometry(factory.createPoint(new Coordinate(25, 32)), "test_pt_2"))
              .get(0);

      final TestGeometry geom2 =
          (TestGeometry)
              mockDataStore
                  .query(new QueryOptions(), new RowIdQuery(Collections.singletonList(rowId2)))
                  .next();

      // as we have chosen to persist the index, we will see the index
      // entry
      // in the index store
      assertEquals(true, indexStore.indexExists(index.getId()));

      // of course, the point is actually stored in this case
      assertEquals("test_pt_2", geom2.id);
    }
  }
Esempio n. 6
0
  private static TreeSet<IntermediateSplitInfo> populateIntermediateSplits(
      final TreeSet<IntermediateSplitInfo> splits,
      final BasicHBaseOperations operations,
      final PrimaryIndex index,
      final List<DataAdapter<Object>> adapters,
      final Map<PrimaryIndex, RowRangeHistogramStatistics<?>> statsCache,
      final AdapterStore adapterStore,
      final DataStatisticsStore statsStore,
      final Integer maxSplits,
      final DistributableQuery query,
      final String[] authorizations)
      throws IOException {

    if ((query != null) && !query.isSupported(index)) {
      return splits;
    }
    final HBaseMRRowRange fullrange = getRangeMax(index, adapterStore, statsStore, authorizations);

    final String tableName = index.getId().getString();
    final NumericIndexStrategy indexStrategy = index.getIndexStrategy();

    // Build list of row ranges from query
    List<HBaseMRRowRange> ranges = new ArrayList<HBaseMRRowRange>();
    final List<ByteArrayRange> constraintRanges;
    if (query != null) {
      final List<MultiDimensionalNumericData> indexConstraints =
          query.getIndexConstraints(indexStrategy);
      if ((maxSplits != null) && (maxSplits > 0)) {
        constraintRanges =
            DataStoreUtils.constraintsToByteArrayRanges(indexConstraints, indexStrategy, maxSplits);
      } else {
        constraintRanges =
            DataStoreUtils.constraintsToByteArrayRanges(indexConstraints, indexStrategy, -1);
      }
      for (final ByteArrayRange constraintRange : constraintRanges) {
        ranges.add(new HBaseMRRowRange(constraintRange));
      }
    } else {
      ranges.add(fullrange);
      if (LOGGER.isTraceEnabled()) {
        LOGGER.trace("Protected range: " + fullrange);
      }
    }

    final Map<HRegionLocation, Map<HRegionInfo, List<HBaseMRRowRange>>> binnedRanges =
        new HashMap<HRegionLocation, Map<HRegionInfo, List<HBaseMRRowRange>>>();
    final RegionLocator regionLocator = operations.getRegionLocator(tableName);
    while (!ranges.isEmpty()) {
      ranges = binRanges(ranges, binnedRanges, regionLocator);
    }

    for (final Entry<HRegionLocation, Map<HRegionInfo, List<HBaseMRRowRange>>> locationEntry :
        binnedRanges.entrySet()) {
      final String hostname = locationEntry.getKey().getHostname();

      for (final Entry<HRegionInfo, List<HBaseMRRowRange>> regionEntry :
          locationEntry.getValue().entrySet()) {
        final Map<PrimaryIndex, List<RangeLocationPair>> splitInfo =
            new HashMap<PrimaryIndex, List<RangeLocationPair>>();
        final List<RangeLocationPair> rangeList = new ArrayList<RangeLocationPair>();

        for (final HBaseMRRowRange range : regionEntry.getValue()) {

          final double cardinality =
              getCardinality(
                  getHistStats(
                      index, adapters, adapterStore, statsStore, statsCache, authorizations),
                  range);

          if (range.intersects(fullrange)) {
            rangeList.add(
                new RangeLocationPair(range, hostname, cardinality < 1 ? 1.0 : cardinality));
          } else {
            LOGGER.info("Query split outside of range");
          }
          if (LOGGER.isTraceEnabled()) {
            LOGGER.warn("Clipped range: " + rangeList.get(rangeList.size() - 1).getRange());
          }
        }

        if (!rangeList.isEmpty()) {
          splitInfo.put(index, rangeList);
          splits.add(new IntermediateSplitInfo(splitInfo));
        }
      }
    }

    return splits;
  }
  /** Initialize a scanner over the given input split using this task attempt configuration. */
  @Override
  public void initialize(final InputSplit inSplit, final TaskAttemptContext attempt)
      throws IOException {
    split = (GeoWaveAccumuloInputSplit) inSplit;

    numKeysRead = 0;

    final Map<RangeLocationPair, CloseableIterator<?>> iteratorsPerRange =
        new LinkedHashMap<RangeLocationPair, CloseableIterator<?>>();

    final Set<PrimaryIndex> indices = split.getIndices();
    BigDecimal sum = BigDecimal.ZERO;

    final Map<RangeLocationPair, BigDecimal> incrementalRangeSums =
        new LinkedHashMap<RangeLocationPair, BigDecimal>();

    for (final PrimaryIndex i : indices) {
      final List<RangeLocationPair> ranges = split.getRanges(i);
      List<QueryFilter> queryFilters = null;
      if (query != null) {
        queryFilters = query.createFilters(i.getIndexModel());
      }
      for (final RangeLocationPair r : ranges) {
        final QueryOptions rangeQueryOptions = new QueryOptions(queryOptions);
        rangeQueryOptions.setIndex(i);
        iteratorsPerRange.put(
            r,
            new InputFormatAccumuloRangeQuery(
                    adapterStore,
                    i,
                    r.getRange(),
                    queryFilters,
                    isOutputWritable,
                    rangeQueryOptions)
                .query(
                    accumuloOperations,
                    adapterStore,
                    rangeQueryOptions.getMaxResolutionSubsamplingPerDimension(),
                    rangeQueryOptions.getLimit()));
        incrementalRangeSums.put(r, sum);
        sum = sum.add(BigDecimal.valueOf(r.getCardinality()));
      }
    }

    // finally we can compute percent progress
    progressPerRange = new LinkedHashMap<RangeLocationPair, ProgressPerRange>();
    RangeLocationPair prevRangeIndex = null;
    float prevProgress = 0f;
    for (final Entry<RangeLocationPair, BigDecimal> entry : incrementalRangeSums.entrySet()) {
      final BigDecimal value = entry.getValue();
      final float progress = value.divide(sum, RoundingMode.HALF_UP).floatValue();
      if (prevRangeIndex != null) {
        progressPerRange.put(prevRangeIndex, new ProgressPerRange(prevProgress, progress));
      }
      prevRangeIndex = entry.getKey();
      prevProgress = progress;
    }
    progressPerRange.put(prevRangeIndex, new ProgressPerRange(prevProgress, 1f));
    // concatenate iterators
    iterator =
        new CloseableIteratorWrapper<Object>(
            new Closeable() {
              @Override
              public void close() throws IOException {
                for (final CloseableIterator<?> it : iteratorsPerRange.values()) {
                  it.close();
                }
              }
            },
            concatenateWithCallback(
                iteratorsPerRange.entrySet().iterator(),
                new NextRangeCallback() {

                  @Override
                  public void setRange(final RangeLocationPair indexPair) {
                    currentGeoWaveRangeIndexPair = indexPair;
                  }
                }));
  }