/** * Test the following constellation: [- index range -] [- graylog_1 -][- graylog_2 -][- graylog_3 * -][- graylog_4 -][- graylog_5 -] */ @Test @UsingDataSet( locations = "EsIndexRangeServiceTest-distinct.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) public void findReturnsIndexRangesWithinGivenRange() throws Exception { final DateTime begin = new DateTime(2015, 1, 2, 12, 0, DateTimeZone.UTC); final DateTime end = new DateTime(2015, 1, 4, 12, 0, DateTimeZone.UTC); final SortedSet<IndexRange> indexRanges = indexRangeService.find(begin, end); assertThat(indexRanges) .containsExactly( IndexRange.create( "graylog_2", new DateTime(2015, 1, 2, 0, 0, DateTimeZone.UTC), new DateTime(2015, 1, 3, 0, 0, DateTimeZone.UTC), new DateTime(2015, 1, 3, 0, 0, DateTimeZone.UTC), 42), IndexRange.create( "graylog_3", new DateTime(2015, 1, 3, 0, 0, DateTimeZone.UTC), new DateTime(2015, 1, 4, 0, 0, DateTimeZone.UTC), new DateTime(2015, 1, 4, 0, 0, DateTimeZone.UTC), 42), IndexRange.create( "graylog_4", new DateTime(2015, 1, 4, 0, 0, DateTimeZone.UTC), new DateTime(2015, 1, 5, 0, 0, DateTimeZone.UTC), new DateTime(2015, 1, 5, 0, 0, DateTimeZone.UTC), 42)); }
@Test @UsingDataSet( locations = "EsIndexRangeServiceTest-EmptyIndex.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) public void testCalculateRangeWithEmptyIndex() throws Exception { final String index = "graylog"; final IndexRange range = indexRangeService.calculateRange(index); assertThat(range).isNotNull(); assertThat(range.indexName()).isEqualTo(index); assertThat(range.begin()).isEqualTo(new DateTime(0L, DateTimeZone.UTC)); assertThat(range.end()).isEqualTo(new DateTime(0L, DateTimeZone.UTC)); }
@Override public void process(JCas aJCas) throws AnalysisEngineProcessException { // Keeping track of the ranges of different relation candidates so we wont have duplicate // text snippets for different candidates on the same text Map<IndexRange, IndexRange> rangeMappings = new HashMap<IndexRange, IndexRange>(); if (aggregateJCas == null) aggregateJCas = getEmptyJCas(); CasCopier copier = new CasCopier(aJCas.getCas(), aggregateJCas.getCas()); Iterator<RelationCandidate> iter = JCasUtil.iterator(aJCas, RelationCandidate.class); while (iter.hasNext()) { RelationCandidate candidate = iter.next(); RelationCandidate candidateCopy = (RelationCandidate) copier.copyFs(candidate); // See if we already have this candidate in the aggregate jcas IndexRange candidateRange = new IndexRange(candidate); // The offset between the old jcas and the new of this relation candidate int offset = 0; if (rangeMappings.containsKey(candidateRange)) { offset = rangeMappings.get(candidateRange).getStart() - candidateRange.getStart(); updateAnnotation(candidateCopy, offset); // No need to copy features, has already been done } else { offset = content.length() - candidateRange.getStart(); updateAnnotation(candidateCopy, offset); rangeMappings.put(candidateRange, new IndexRange(candidateCopy)); // For every feature we want to copy for (Class<? extends Annotation> feature : features) { // Iterating over the annotations of this feature type covered by this relation candidate for (Annotation annotation : JCasUtil.selectCovered(aJCas, feature, candidate)) { Annotation cAnnotation = (Annotation) copier.copyFs(annotation); // Updating the indices of the annotation updateAnnotation(cAnnotation, offset); aggregateJCas.addFsToIndexes(cAnnotation); } } // Adding the text content of the relation candidate to the new cas content.append(candidate.getCoveredText()); } aggregateJCas.addFsToIndexes(candidateCopy); } }
@Test @UsingDataSet( locations = "EsIndexRangeServiceTest.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) public void calculateRangeReturnsIndexRange() throws Exception { final String index = "graylog"; final DateTime min = new DateTime(2015, 1, 1, 1, 0, DateTimeZone.UTC); final DateTime max = new DateTime(2015, 1, 1, 5, 0, DateTimeZone.UTC); final IndexRange indexRange = indexRangeService.calculateRange(index); assertThat(indexRange.indexName()).isEqualTo(index); assertThat(indexRange.begin()).isEqualTo(min); assertThat(indexRange.end()).isEqualTo(max); Assertions.assertThat(indexRange.calculatedAt()) .isEqualToIgnoringHours(DateTime.now(DateTimeZone.UTC)); }
@Test @UsingDataSet( locations = "EsIndexRangeServiceTest.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) public void savePersistsIndexRangeInReadOnlyIndex() throws Exception { final String indexName = "graylog_read_only"; final DateTime begin = new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC); final DateTime end = new DateTime(2015, 1, 2, 0, 0, DateTimeZone.UTC); final DateTime now = DateTime.now(DateTimeZone.UTC); final IndexRange indexRange = IndexRange.create(indexName, begin, end, now, 42); try { indices.create(indexName); indices.setReadOnly(indexName); assumeTrue(indices.isReadOnly(indexName)); indexRangeService.save(indexRange); verify(clusterEventBus, times(1)).post(IndexRangeUpdatedEvent.create(indexName)); assertThat(indices.isReadOnly(indexName)).isTrue(); final IndexRange result = indexRangeService.get(indexName); assertThat(result.indexName()).isEqualTo(indexName); assertThat(result.begin()).isEqualTo(begin); assertThat(result.end()).isEqualTo(end); assertThat(result.calculatedAt()).isEqualTo(now); assertThat(result.calculationDuration()).isEqualTo(42); } finally { indices.delete(indexName); } }
protected Polygon<?> createGeometry(SqlServerGeometry nativeGeom, int shapeIndex) { if (nativeGeom.isEmptyShape(shapeIndex)) { return (Polygon<?>) createNullGeometry(nativeGeom.getCoordinateReferenceSystem()); } // polygons consist of one exterior ring figure, and several interior ones. IndexRange figureRange = nativeGeom.getFiguresForShape(shapeIndex); List<LinearRing<?>> rings = new ArrayList<LinearRing<?>>(figureRange.length()); // the rings should contain all inner rings from index 1 to index length - 1 // index = 0 should be reserved for the shell. for (int figureIdx = figureRange.start, i = 1; figureIdx < figureRange.end; figureIdx++) { IndexRange pntIndexRange = nativeGeom.getPointsForFigure(figureIdx); if (nativeGeom.isFigureInteriorRing(figureIdx)) { rings.set(i++, toLinearRing(nativeGeom, pntIndexRange)); } else { rings.set(0, toLinearRing(nativeGeom, pntIndexRange)); } } LinearRing<?>[] ringArr = (LinearRing<?>[]) new LinearRing[rings.size()]; return new Polygon(rings.toArray(ringArr)); }
@Test @UsingDataSet(loadStrategy = LoadStrategyEnum.CLEAN_INSERT) public void saveSkipsUpToDateExistingIndexRange() throws Exception { final String indexName = "graylog_1"; final DateTime begin = new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC); final DateTime end = new DateTime(2015, 1, 2, 0, 0, DateTimeZone.UTC); final DateTime calculatedAt = new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC); final IndexRange indexRange = IndexRange.create(indexName, begin, end, calculatedAt, 23); indexRangeService.save(indexRange); verify(clusterEventBus, never()).post(IndexRangeUpdatedEvent.create(indexName)); }
/** * Performs a binary search for {@code value} in the ascending sorted off-heap struct collection * using int struct key. Returns range of indices having given value or empty range. Searching in * an unsorted collection has an undefined result. It's also undefined which element is found if * there are multiple occurrences of the same element. * * @param collection the sorted array to search. * @param startIndex the inclusive start index. * @param endIndex the exclusive end index. * @param value the element to find. * @param keyOffset int key field offset within stuct bounds * @param out range instance, will be set with start/end indices having given value or with empty * value */ public static void binarySearchRangeByIntKey( OffHeapStructCollection collection, long startIndex, long endIndex, long value, int keyOffset, IndexRange out) { long ind = binarySearchByIntKey(collection, startIndex, endIndex, value, keyOffset); if (ind < 0) { out.setEmpty(ind); return; } long from = ind; while (from >= startIndex && value == collection.getInt(from, keyOffset)) { from -= 1; } from += 1; long to = ind; while (to < endIndex && value == collection.getInt(to, keyOffset)) { to += 1; } to -= 1; out.set(from, to); }
/* * (non-Javadoc) * @see com.lexst.db.view.View#add(com.lexst.util.host.SiteHost, com.lexst.db.sign.range.SignRange) */ @Override public boolean add(SiteHost host, IndexRange index) { if (index.getClass() != ShortIndexRange.class) { throw new ClassCastException("not small index"); } ShortIndexRange idx = (ShortIndexRange) index; short begin = idx.getBegin(); short end = idx.getEnd(); ShortRange range = new ShortRange(begin, end); long chunkId = idx.getChunkId(); IdentitySet set = mapSet.get(range); if (set == null) { set = new IdentitySet(); mapSet.put(range, set); } return set.add(host, chunkId); }
@Test @UsingDataSet( locations = "EsIndexRangeServiceTest.json", loadStrategy = LoadStrategyEnum.CLEAN_INSERT) public void getReturnsExistingIndexRange() throws Exception { IndexRange indexRange = indexRangeService.get("graylog_1"); assertThat(indexRange.indexName()).isEqualTo("graylog_1"); assertThat(indexRange.begin()).isEqualTo(new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC)); assertThat(indexRange.end()).isEqualTo(new DateTime(2015, 1, 2, 0, 0, DateTimeZone.UTC)); assertThat(indexRange.calculatedAt()) .isEqualTo(new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC)); assertThat(indexRange.calculationDuration()).isEqualTo(23); }
@Test @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) public void savePersistsIndexRange() throws Exception { final String indexName = "graylog"; final DateTime begin = new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC); final DateTime end = new DateTime(2015, 1, 2, 0, 0, DateTimeZone.UTC); final DateTime now = DateTime.now(DateTimeZone.UTC); final IndexRange indexRange = IndexRange.create(indexName, begin, end, now, 42); indexRangeService.save(indexRange); final IndexRange result = indexRangeService.get(indexName); verify(clusterEventBus, times(1)).post(IndexRangeUpdatedEvent.create(indexName)); assertThat(result.indexName()).isEqualTo(indexName); assertThat(result.begin()).isEqualTo(begin); assertThat(result.end()).isEqualTo(end); assertThat(result.calculatedAt()).isEqualTo(now); assertThat(result.calculationDuration()).isEqualTo(42); }
// Reused by both MR and Spark for performing zero out public static IndexRange getSelectedRangeForZeroOut( IndexedMatrixValue in, int blockRowFactor, int blockColFactor, IndexRange indexRange) { IndexRange tempRange = new IndexRange(-1, -1, -1, -1); long topBlockRowIndex = UtilFunctions.computeBlockIndex(indexRange.rowStart, blockRowFactor); int topRowInTopBlock = UtilFunctions.computeCellInBlock(indexRange.rowStart, blockRowFactor); long bottomBlockRowIndex = UtilFunctions.computeBlockIndex(indexRange.rowEnd, blockRowFactor); int bottomRowInBottomBlock = UtilFunctions.computeCellInBlock(indexRange.rowEnd, blockRowFactor); long leftBlockColIndex = UtilFunctions.computeBlockIndex(indexRange.colStart, blockColFactor); int leftColInLeftBlock = UtilFunctions.computeCellInBlock(indexRange.colStart, blockColFactor); long rightBlockColIndex = UtilFunctions.computeBlockIndex(indexRange.colEnd, blockColFactor); int rightColInRightBlock = UtilFunctions.computeCellInBlock(indexRange.colEnd, blockColFactor); // no overlap if (in.getIndexes().getRowIndex() < topBlockRowIndex || in.getIndexes().getRowIndex() > bottomBlockRowIndex || in.getIndexes().getColumnIndex() < leftBlockColIndex || in.getIndexes().getColumnIndex() > rightBlockColIndex) { tempRange.set(-1, -1, -1, -1); return tempRange; } // get the index range inside the block tempRange.set(0, in.getValue().getNumRows() - 1, 0, in.getValue().getNumColumns() - 1); if (topBlockRowIndex == in.getIndexes().getRowIndex()) tempRange.rowStart = topRowInTopBlock; if (bottomBlockRowIndex == in.getIndexes().getRowIndex()) tempRange.rowEnd = bottomRowInBottomBlock; if (leftBlockColIndex == in.getIndexes().getColumnIndex()) tempRange.colStart = leftColInLeftBlock; if (rightBlockColIndex == in.getIndexes().getColumnIndex()) tempRange.colEnd = rightColInRightBlock; return tempRange; }
@Test @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL) public void saveOverwritesExistingIndexRange() throws Exception { final String indexName = "graylog"; final DateTime begin = new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC); final DateTime end = new DateTime(2015, 1, 2, 0, 0, DateTimeZone.UTC); final DateTime now = DateTime.now(DateTimeZone.UTC); final IndexRange indexRangeBefore = IndexRange.create(indexName, begin, end, now, 1); final IndexRange indexRangeAfter = IndexRange.create(indexName, begin, end, now, 2); indexRangeService.save(indexRangeBefore); final IndexRange before = indexRangeService.get(indexName); assertThat(before.calculationDuration()).isEqualTo(1); indexRangeService.save(indexRangeAfter); final IndexRange after = indexRangeService.get(indexName); assertThat(after.calculationDuration()).isEqualTo(2); verify(clusterEventBus, times(2)).post(IndexRangeUpdatedEvent.create(indexName)); }
public void setRange(int begin, int end) { mRange.begin = begin; mRange.end = end; mPos = begin - 1; }