/**
   * Test the following constellation: [- index range -] [- graylog_1 -][- graylog_2 -][- graylog_3
   * -][- graylog_4 -][- graylog_5 -]
   */
  @Test
  @UsingDataSet(
      locations = "EsIndexRangeServiceTest-distinct.json",
      loadStrategy = LoadStrategyEnum.CLEAN_INSERT)
  public void findReturnsIndexRangesWithinGivenRange() throws Exception {
    final DateTime begin = new DateTime(2015, 1, 2, 12, 0, DateTimeZone.UTC);
    final DateTime end = new DateTime(2015, 1, 4, 12, 0, DateTimeZone.UTC);
    final SortedSet<IndexRange> indexRanges = indexRangeService.find(begin, end);

    assertThat(indexRanges)
        .containsExactly(
            IndexRange.create(
                "graylog_2",
                new DateTime(2015, 1, 2, 0, 0, DateTimeZone.UTC),
                new DateTime(2015, 1, 3, 0, 0, DateTimeZone.UTC),
                new DateTime(2015, 1, 3, 0, 0, DateTimeZone.UTC),
                42),
            IndexRange.create(
                "graylog_3",
                new DateTime(2015, 1, 3, 0, 0, DateTimeZone.UTC),
                new DateTime(2015, 1, 4, 0, 0, DateTimeZone.UTC),
                new DateTime(2015, 1, 4, 0, 0, DateTimeZone.UTC),
                42),
            IndexRange.create(
                "graylog_4",
                new DateTime(2015, 1, 4, 0, 0, DateTimeZone.UTC),
                new DateTime(2015, 1, 5, 0, 0, DateTimeZone.UTC),
                new DateTime(2015, 1, 5, 0, 0, DateTimeZone.UTC),
                42));
  }
  @Test
  @UsingDataSet(
      locations = "EsIndexRangeServiceTest-EmptyIndex.json",
      loadStrategy = LoadStrategyEnum.CLEAN_INSERT)
  public void testCalculateRangeWithEmptyIndex() throws Exception {
    final String index = "graylog";
    final IndexRange range = indexRangeService.calculateRange(index);

    assertThat(range).isNotNull();
    assertThat(range.indexName()).isEqualTo(index);
    assertThat(range.begin()).isEqualTo(new DateTime(0L, DateTimeZone.UTC));
    assertThat(range.end()).isEqualTo(new DateTime(0L, DateTimeZone.UTC));
  }
  @Override
  public void process(JCas aJCas) throws AnalysisEngineProcessException {

    // Keeping track of the ranges of different relation candidates so we wont have duplicate
    // text snippets for different candidates on the same text
    Map<IndexRange, IndexRange> rangeMappings = new HashMap<IndexRange, IndexRange>();

    if (aggregateJCas == null) aggregateJCas = getEmptyJCas();

    CasCopier copier = new CasCopier(aJCas.getCas(), aggregateJCas.getCas());
    Iterator<RelationCandidate> iter = JCasUtil.iterator(aJCas, RelationCandidate.class);

    while (iter.hasNext()) {

      RelationCandidate candidate = iter.next();
      RelationCandidate candidateCopy = (RelationCandidate) copier.copyFs(candidate);

      // See if we already have this candidate in the aggregate jcas
      IndexRange candidateRange = new IndexRange(candidate);
      // The offset between the old jcas and the new of this relation candidate
      int offset = 0;

      if (rangeMappings.containsKey(candidateRange)) {
        offset = rangeMappings.get(candidateRange).getStart() - candidateRange.getStart();
        updateAnnotation(candidateCopy, offset);
        // No need to copy features, has already been done

      } else {
        offset = content.length() - candidateRange.getStart();
        updateAnnotation(candidateCopy, offset);
        rangeMappings.put(candidateRange, new IndexRange(candidateCopy));

        // For every feature we want to copy
        for (Class<? extends Annotation> feature : features) {

          // Iterating over the annotations of this feature type covered by this relation candidate
          for (Annotation annotation : JCasUtil.selectCovered(aJCas, feature, candidate)) {
            Annotation cAnnotation = (Annotation) copier.copyFs(annotation);
            // Updating the indices of the annotation
            updateAnnotation(cAnnotation, offset);
            aggregateJCas.addFsToIndexes(cAnnotation);
          }
        }
        // Adding the text content of the relation candidate to the new cas
        content.append(candidate.getCoveredText());
      }
      aggregateJCas.addFsToIndexes(candidateCopy);
    }
  }
  @Test
  @UsingDataSet(
      locations = "EsIndexRangeServiceTest.json",
      loadStrategy = LoadStrategyEnum.CLEAN_INSERT)
  public void calculateRangeReturnsIndexRange() throws Exception {
    final String index = "graylog";
    final DateTime min = new DateTime(2015, 1, 1, 1, 0, DateTimeZone.UTC);
    final DateTime max = new DateTime(2015, 1, 1, 5, 0, DateTimeZone.UTC);
    final IndexRange indexRange = indexRangeService.calculateRange(index);

    assertThat(indexRange.indexName()).isEqualTo(index);
    assertThat(indexRange.begin()).isEqualTo(min);
    assertThat(indexRange.end()).isEqualTo(max);
    Assertions.assertThat(indexRange.calculatedAt())
        .isEqualToIgnoringHours(DateTime.now(DateTimeZone.UTC));
  }
  @Test
  @UsingDataSet(
      locations = "EsIndexRangeServiceTest.json",
      loadStrategy = LoadStrategyEnum.CLEAN_INSERT)
  public void savePersistsIndexRangeInReadOnlyIndex() throws Exception {
    final String indexName = "graylog_read_only";
    final DateTime begin = new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC);
    final DateTime end = new DateTime(2015, 1, 2, 0, 0, DateTimeZone.UTC);
    final DateTime now = DateTime.now(DateTimeZone.UTC);
    final IndexRange indexRange = IndexRange.create(indexName, begin, end, now, 42);

    try {
      indices.create(indexName);
      indices.setReadOnly(indexName);
      assumeTrue(indices.isReadOnly(indexName));
      indexRangeService.save(indexRange);

      verify(clusterEventBus, times(1)).post(IndexRangeUpdatedEvent.create(indexName));
      assertThat(indices.isReadOnly(indexName)).isTrue();

      final IndexRange result = indexRangeService.get(indexName);
      assertThat(result.indexName()).isEqualTo(indexName);
      assertThat(result.begin()).isEqualTo(begin);
      assertThat(result.end()).isEqualTo(end);
      assertThat(result.calculatedAt()).isEqualTo(now);
      assertThat(result.calculationDuration()).isEqualTo(42);
    } finally {
      indices.delete(indexName);
    }
  }
 protected Polygon<?> createGeometry(SqlServerGeometry nativeGeom, int shapeIndex) {
   if (nativeGeom.isEmptyShape(shapeIndex)) {
     return (Polygon<?>) createNullGeometry(nativeGeom.getCoordinateReferenceSystem());
   }
   // polygons consist of one exterior ring figure, and several interior ones.
   IndexRange figureRange = nativeGeom.getFiguresForShape(shapeIndex);
   List<LinearRing<?>> rings = new ArrayList<LinearRing<?>>(figureRange.length());
   // the rings should contain all inner rings from index 1 to index length - 1
   // index = 0 should be reserved for the shell.
   for (int figureIdx = figureRange.start, i = 1; figureIdx < figureRange.end; figureIdx++) {
     IndexRange pntIndexRange = nativeGeom.getPointsForFigure(figureIdx);
     if (nativeGeom.isFigureInteriorRing(figureIdx)) {
       rings.set(i++, toLinearRing(nativeGeom, pntIndexRange));
     } else {
       rings.set(0, toLinearRing(nativeGeom, pntIndexRange));
     }
   }
   LinearRing<?>[] ringArr = (LinearRing<?>[]) new LinearRing[rings.size()];
   return new Polygon(rings.toArray(ringArr));
 }
  @Test
  @UsingDataSet(loadStrategy = LoadStrategyEnum.CLEAN_INSERT)
  public void saveSkipsUpToDateExistingIndexRange() throws Exception {
    final String indexName = "graylog_1";
    final DateTime begin = new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC);
    final DateTime end = new DateTime(2015, 1, 2, 0, 0, DateTimeZone.UTC);
    final DateTime calculatedAt = new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC);
    final IndexRange indexRange = IndexRange.create(indexName, begin, end, calculatedAt, 23);

    indexRangeService.save(indexRange);
    verify(clusterEventBus, never()).post(IndexRangeUpdatedEvent.create(indexName));
  }
 /**
  * Performs a binary search for {@code value} in the ascending sorted off-heap struct collection
  * using int struct key. Returns range of indices having given value or empty range. Searching in
  * an unsorted collection has an undefined result. It's also undefined which element is found if
  * there are multiple occurrences of the same element.
  *
  * @param collection the sorted array to search.
  * @param startIndex the inclusive start index.
  * @param endIndex the exclusive end index.
  * @param value the element to find.
  * @param keyOffset int key field offset within stuct bounds
  * @param out range instance, will be set with start/end indices having given value or with empty
  *     value
  */
 public static void binarySearchRangeByIntKey(
     OffHeapStructCollection collection,
     long startIndex,
     long endIndex,
     long value,
     int keyOffset,
     IndexRange out) {
   long ind = binarySearchByIntKey(collection, startIndex, endIndex, value, keyOffset);
   if (ind < 0) {
     out.setEmpty(ind);
     return;
   }
   long from = ind;
   while (from >= startIndex && value == collection.getInt(from, keyOffset)) {
     from -= 1;
   }
   from += 1;
   long to = ind;
   while (to < endIndex && value == collection.getInt(to, keyOffset)) {
     to += 1;
   }
   to -= 1;
   out.set(from, to);
 }
Exemple #9
0
  /*
   * (non-Javadoc)
   * @see com.lexst.db.view.View#add(com.lexst.util.host.SiteHost, com.lexst.db.sign.range.SignRange)
   */
  @Override
  public boolean add(SiteHost host, IndexRange index) {
    if (index.getClass() != ShortIndexRange.class) {
      throw new ClassCastException("not small index");
    }
    ShortIndexRange idx = (ShortIndexRange) index;
    short begin = idx.getBegin();
    short end = idx.getEnd();
    ShortRange range = new ShortRange(begin, end);
    long chunkId = idx.getChunkId();

    IdentitySet set = mapSet.get(range);
    if (set == null) {
      set = new IdentitySet();
      mapSet.put(range, set);
    }
    return set.add(host, chunkId);
  }
  @Test
  @UsingDataSet(
      locations = "EsIndexRangeServiceTest.json",
      loadStrategy = LoadStrategyEnum.CLEAN_INSERT)
  public void getReturnsExistingIndexRange() throws Exception {
    IndexRange indexRange = indexRangeService.get("graylog_1");

    assertThat(indexRange.indexName()).isEqualTo("graylog_1");
    assertThat(indexRange.begin()).isEqualTo(new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC));
    assertThat(indexRange.end()).isEqualTo(new DateTime(2015, 1, 2, 0, 0, DateTimeZone.UTC));
    assertThat(indexRange.calculatedAt())
        .isEqualTo(new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC));
    assertThat(indexRange.calculationDuration()).isEqualTo(23);
  }
  @Test
  @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL)
  public void savePersistsIndexRange() throws Exception {
    final String indexName = "graylog";
    final DateTime begin = new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC);
    final DateTime end = new DateTime(2015, 1, 2, 0, 0, DateTimeZone.UTC);
    final DateTime now = DateTime.now(DateTimeZone.UTC);
    final IndexRange indexRange = IndexRange.create(indexName, begin, end, now, 42);

    indexRangeService.save(indexRange);

    final IndexRange result = indexRangeService.get(indexName);
    verify(clusterEventBus, times(1)).post(IndexRangeUpdatedEvent.create(indexName));
    assertThat(result.indexName()).isEqualTo(indexName);
    assertThat(result.begin()).isEqualTo(begin);
    assertThat(result.end()).isEqualTo(end);
    assertThat(result.calculatedAt()).isEqualTo(now);
    assertThat(result.calculationDuration()).isEqualTo(42);
  }
  // Reused by both MR and Spark for performing zero out
  public static IndexRange getSelectedRangeForZeroOut(
      IndexedMatrixValue in, int blockRowFactor, int blockColFactor, IndexRange indexRange) {
    IndexRange tempRange = new IndexRange(-1, -1, -1, -1);

    long topBlockRowIndex = UtilFunctions.computeBlockIndex(indexRange.rowStart, blockRowFactor);
    int topRowInTopBlock = UtilFunctions.computeCellInBlock(indexRange.rowStart, blockRowFactor);
    long bottomBlockRowIndex = UtilFunctions.computeBlockIndex(indexRange.rowEnd, blockRowFactor);
    int bottomRowInBottomBlock =
        UtilFunctions.computeCellInBlock(indexRange.rowEnd, blockRowFactor);

    long leftBlockColIndex = UtilFunctions.computeBlockIndex(indexRange.colStart, blockColFactor);
    int leftColInLeftBlock = UtilFunctions.computeCellInBlock(indexRange.colStart, blockColFactor);
    long rightBlockColIndex = UtilFunctions.computeBlockIndex(indexRange.colEnd, blockColFactor);
    int rightColInRightBlock = UtilFunctions.computeCellInBlock(indexRange.colEnd, blockColFactor);

    // no overlap
    if (in.getIndexes().getRowIndex() < topBlockRowIndex
        || in.getIndexes().getRowIndex() > bottomBlockRowIndex
        || in.getIndexes().getColumnIndex() < leftBlockColIndex
        || in.getIndexes().getColumnIndex() > rightBlockColIndex) {
      tempRange.set(-1, -1, -1, -1);
      return tempRange;
    }

    // get the index range inside the block
    tempRange.set(0, in.getValue().getNumRows() - 1, 0, in.getValue().getNumColumns() - 1);
    if (topBlockRowIndex == in.getIndexes().getRowIndex()) tempRange.rowStart = topRowInTopBlock;
    if (bottomBlockRowIndex == in.getIndexes().getRowIndex())
      tempRange.rowEnd = bottomRowInBottomBlock;
    if (leftBlockColIndex == in.getIndexes().getColumnIndex())
      tempRange.colStart = leftColInLeftBlock;
    if (rightBlockColIndex == in.getIndexes().getColumnIndex())
      tempRange.colEnd = rightColInRightBlock;

    return tempRange;
  }
  @Test
  @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL)
  public void saveOverwritesExistingIndexRange() throws Exception {
    final String indexName = "graylog";
    final DateTime begin = new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC);
    final DateTime end = new DateTime(2015, 1, 2, 0, 0, DateTimeZone.UTC);
    final DateTime now = DateTime.now(DateTimeZone.UTC);
    final IndexRange indexRangeBefore = IndexRange.create(indexName, begin, end, now, 1);
    final IndexRange indexRangeAfter = IndexRange.create(indexName, begin, end, now, 2);

    indexRangeService.save(indexRangeBefore);

    final IndexRange before = indexRangeService.get(indexName);
    assertThat(before.calculationDuration()).isEqualTo(1);

    indexRangeService.save(indexRangeAfter);

    final IndexRange after = indexRangeService.get(indexName);
    assertThat(after.calculationDuration()).isEqualTo(2);

    verify(clusterEventBus, times(2)).post(IndexRangeUpdatedEvent.create(indexName));
  }
 public void setRange(int begin, int end) {
   mRange.begin = begin;
   mRange.end = end;
   mPos = begin - 1;
 }