コード例 #1
0
ファイル: HBaseMRUtils.java プロジェクト: CameronWork/geowave
  /** Read the metadata table to get tablets and match up ranges to them. */
  public static List<InputSplit> getSplits(
      final BasicHBaseOperations operations,
      final DistributableQuery query,
      final QueryOptions queryOptions,
      final AdapterStore adapterStore,
      final DataStatisticsStore statsStore,
      final IndexStore indexStore,
      final AdapterIndexMappingStore adapterIndexMappingStore,
      final Integer minSplits,
      final Integer maxSplits)
      throws IOException, InterruptedException {

    final Map<PrimaryIndex, RowRangeHistogramStatistics<?>> statsCache =
        new HashMap<PrimaryIndex, RowRangeHistogramStatistics<?>>();

    final List<InputSplit> retVal = new ArrayList<InputSplit>();
    final TreeSet<IntermediateSplitInfo> splits = new TreeSet<IntermediateSplitInfo>();

    for (final Pair<PrimaryIndex, List<DataAdapter<Object>>> indexAdapterPair :
        queryOptions.getAdaptersWithMinimalSetOfIndices(
            adapterStore, adapterIndexMappingStore, indexStore)) {

      populateIntermediateSplits(
          splits,
          operations,
          indexAdapterPair.getLeft(),
          indexAdapterPair.getValue(),
          statsCache,
          adapterStore,
          statsStore,
          maxSplits,
          query,
          queryOptions.getAuthorizations());
    }

    // this is an incremental algorithm, it may be better use the target
    // split count to drive it (ie. to get 3 splits this will split 1
    // large
    // range into two down the middle and then split one of those ranges
    // down the middle to get 3, rather than splitting one range into
    // thirds)
    if (!statsCache.isEmpty()
        && !splits.isEmpty()
        && (minSplits != null)
        && (splits.size() < minSplits)) {
      // set the ranges to at least min splits
      do {
        // remove the highest range, split it into 2 and add both
        // back,
        // increasing the size by 1
        final IntermediateSplitInfo highestSplit = splits.pollLast();
        final IntermediateSplitInfo otherSplit = highestSplit.split(statsCache);
        splits.add(highestSplit);
        if (otherSplit == null) {
          LOGGER.warn("Cannot meet minimum splits");
          break;
        }
        splits.add(otherSplit);
      } while (splits.size() < minSplits);
    } else if (((maxSplits != null) && (maxSplits > 0)) && (splits.size() > maxSplits)) {
      // merge splits to fit within max splits
      do {
        // this is the naive approach, remove the lowest two ranges
        // and
        // merge them, decreasing the size by 1

        // TODO Ideally merge takes into account locations (as well
        // as
        // possibly the index as a secondary criteria) to limit the
        // number of locations/indices
        final IntermediateSplitInfo lowestSplit = splits.pollFirst();
        final IntermediateSplitInfo nextLowestSplit = splits.pollFirst();
        lowestSplit.merge(nextLowestSplit);
        splits.add(lowestSplit);
      } while (splits.size() > maxSplits);
    }

    for (final IntermediateSplitInfo split : splits) {
      retVal.add(split.toFinalSplit());
    }
    return retVal;
  }
コード例 #2
0
  /** Initialize a scanner over the given input split using this task attempt configuration. */
  @Override
  public void initialize(final InputSplit inSplit, final TaskAttemptContext attempt)
      throws IOException {
    split = (GeoWaveAccumuloInputSplit) inSplit;

    numKeysRead = 0;

    final Map<RangeLocationPair, CloseableIterator<?>> iteratorsPerRange =
        new LinkedHashMap<RangeLocationPair, CloseableIterator<?>>();

    final Set<PrimaryIndex> indices = split.getIndices();
    BigDecimal sum = BigDecimal.ZERO;

    final Map<RangeLocationPair, BigDecimal> incrementalRangeSums =
        new LinkedHashMap<RangeLocationPair, BigDecimal>();

    for (final PrimaryIndex i : indices) {
      final List<RangeLocationPair> ranges = split.getRanges(i);
      List<QueryFilter> queryFilters = null;
      if (query != null) {
        queryFilters = query.createFilters(i.getIndexModel());
      }
      for (final RangeLocationPair r : ranges) {
        final QueryOptions rangeQueryOptions = new QueryOptions(queryOptions);
        rangeQueryOptions.setIndex(i);
        iteratorsPerRange.put(
            r,
            new InputFormatAccumuloRangeQuery(
                    adapterStore,
                    i,
                    r.getRange(),
                    queryFilters,
                    isOutputWritable,
                    rangeQueryOptions)
                .query(
                    accumuloOperations,
                    adapterStore,
                    rangeQueryOptions.getMaxResolutionSubsamplingPerDimension(),
                    rangeQueryOptions.getLimit()));
        incrementalRangeSums.put(r, sum);
        sum = sum.add(BigDecimal.valueOf(r.getCardinality()));
      }
    }

    // finally we can compute percent progress
    progressPerRange = new LinkedHashMap<RangeLocationPair, ProgressPerRange>();
    RangeLocationPair prevRangeIndex = null;
    float prevProgress = 0f;
    for (final Entry<RangeLocationPair, BigDecimal> entry : incrementalRangeSums.entrySet()) {
      final BigDecimal value = entry.getValue();
      final float progress = value.divide(sum, RoundingMode.HALF_UP).floatValue();
      if (prevRangeIndex != null) {
        progressPerRange.put(prevRangeIndex, new ProgressPerRange(prevProgress, progress));
      }
      prevRangeIndex = entry.getKey();
      prevProgress = progress;
    }
    progressPerRange.put(prevRangeIndex, new ProgressPerRange(prevProgress, 1f));
    // concatenate iterators
    iterator =
        new CloseableIteratorWrapper<Object>(
            new Closeable() {
              @Override
              public void close() throws IOException {
                for (final CloseableIterator<?> it : iteratorsPerRange.values()) {
                  it.close();
                }
              }
            },
            concatenateWithCallback(
                iteratorsPerRange.entrySet().iterator(),
                new NextRangeCallback() {

                  @Override
                  public void setRange(final RangeLocationPair indexPair) {
                    currentGeoWaveRangeIndexPair = indexPair;
                  }
                }));
  }