/**
* this method will call initialize for each message, since we are caching the entity indexes, we
* don't worry about aggregating by app id
*
* @param indexOperationMessage
*/
private void initializeEntityIndexes(final IndexOperationMessage indexOperationMessage) {
// create a set so we can have a unique list of appIds for which we call createEntityIndex
Set<UUID> appIds = new HashSet<>();
// loop through all indexRequests and add the appIds to the set
indexOperationMessage
.getIndexRequests()
.forEach(
req -> {
UUID appId = IndexingUtils.getApplicationIdFromIndexDocId(req.documentId);
appIds.add(appId);
});
// loop through all deindexRequests and add the appIds to the set
indexOperationMessage
.getDeIndexRequests()
.forEach(
req -> {
UUID appId = IndexingUtils.getApplicationIdFromIndexDocId(req.documentId);
appIds.add(appId);
});
// for each of the appIds in the unique set, call create entity index to ensure the aliases are
// created
appIds.forEach(
appId -> {
ApplicationScope appScope = CpNamingUtils.getApplicationScope(appId);
entityIndexFactory.createEntityIndex(
indexLocationStrategyFactory.getIndexLocationStrategy(appScope));
});
}
@Test
public void testDeletes() throws Exception {
EntityManager entityManager = this.app.getEntityManager();
Map<String, Object> map = new HashMap<>();
for (int i = 0; i < 10; i++) {
map.put("somekey", UUID.randomUUID());
Entity entity = entityManager.create("tests", map);
}
this.app.refreshIndex();
Thread.sleep(500);
ApplicationScope appScope = CpNamingUtils.getApplicationScope(entityManager.getApplicationId());
Observable<Id> ids = this.app.getApplicationService().deleteAllEntities(appScope, 5);
int count = ids.count().toBlocking().last();
Assert.assertEquals(count, 5);
ids = this.app.getApplicationService().deleteAllEntities(appScope, 5);
count = ids.count().toBlocking().last();
Assert.assertEquals(count, 5);
this.app.refreshIndex();
Thread.sleep(5000);
Injector injector = SpringResource.getInstance().getBean(Injector.class);
GraphManagerFactory factory = injector.getInstance(GraphManagerFactory.class);
GraphManager graphManager = factory.createEdgeManager(appScope);
SimpleSearchByEdgeType simpleSearchByEdgeType =
new SimpleSearchByEdgeType(
appScope.getApplication(),
CpNamingUtils.getEdgeTypeFromCollectionName("tests"),
Long.MAX_VALUE,
SearchByEdgeType.Order.DESCENDING,
Optional.<Edge>absent());
Iterator<Edge> results =
graphManager.loadEdgesFromSource(simpleSearchByEdgeType).toBlocking().getIterator();
if (results.hasNext()) {
Assert.fail("should be empty");
} else {
Results searchCollection =
entityManager.searchCollection(entityManager.getApplication(), "tests", Query.all());
Assert.assertEquals(searchCollection.size(), 0);
AggregationServiceFactory aggregationServiceFactory =
injector.getInstance(AggregationServiceFactory.class);
long size =
aggregationServiceFactory.getAggregationService().getCollectionSize(appScope, "tests");
Assert.assertEquals(size, 0);
// success
}
}
/** Get the map manager for uuid mapping */
private MapManager getMapManagerForTypes(ApplicationScope applicationScope) {
Id mapOwner = new SimpleId(applicationScope.getApplication().getUuid(), TYPE_APPLICATION);
final MapScope ms = CpNamingUtils.getEntityTypeMapScope(mapOwner);
MapManager mm = mapManagerFactory.createMapManager(ms);
return mm;
}
@Inject
public AsyncEventServiceImpl(
final QueueManagerFactory queueManagerFactory,
final IndexProcessorFig indexProcessorFig,
final IndexProducer indexProducer,
final MetricsFactory metricsFactory,
final EntityCollectionManagerFactory entityCollectionManagerFactory,
final IndexLocationStrategyFactory indexLocationStrategyFactory,
final EntityIndexFactory entityIndexFactory,
final EventBuilder eventBuilder,
final MapManagerFactory mapManagerFactory,
final QueueFig queueFig,
@EventExecutionScheduler final RxTaskScheduler rxTaskScheduler) {
this.indexProducer = indexProducer;
this.entityCollectionManagerFactory = entityCollectionManagerFactory;
this.indexLocationStrategyFactory = indexLocationStrategyFactory;
this.entityIndexFactory = entityIndexFactory;
this.eventBuilder = eventBuilder;
final MapScope mapScope =
new MapScopeImpl(CpNamingUtils.getManagementApplicationId(), "indexEvents");
this.esMapPersistence = mapManagerFactory.createMapManager(mapScope);
this.rxTaskScheduler = rxTaskScheduler;
QueueScope queueScope = new QueueScopeImpl(QUEUE_NAME, QueueScope.RegionImplementation.ALL);
this.queue = queueManagerFactory.getQueueManager(queueScope);
this.indexProcessorFig = indexProcessorFig;
this.queueFig = queueFig;
this.writeTimer = metricsFactory.getTimer(AsyncEventServiceImpl.class, "async_event.write");
this.readTimer = metricsFactory.getTimer(AsyncEventServiceImpl.class, "async_event.read");
this.ackTimer = metricsFactory.getTimer(AsyncEventServiceImpl.class, "async_event.ack");
this.indexErrorCounter =
metricsFactory.getCounter(AsyncEventServiceImpl.class, "async_event.error");
this.messageCycle =
metricsFactory.getHistogram(AsyncEventServiceImpl.class, "async_event.message_cycle");
// wire up the gauge of inflight message
metricsFactory.addGauge(
AsyncEventServiceImpl.class,
"async-event.inflight",
new Gauge<Long>() {
@Override
public Long getValue() {
return inFlight.longValue();
}
});
start();
}
@Override
protected String getEdgeName() {
return CpNamingUtils.getEdgeTypeFromConnectionType(connectionName);
}
@Singleton
public class ReIndexServiceImpl implements ReIndexService {
private static final Logger logger = LoggerFactory.getLogger(ReIndexServiceImpl.class);
private static final MapScope RESUME_MAP_SCOPE =
new MapScopeImpl(CpNamingUtils.getManagementApplicationId(), "reindexresume");
// Keep cursors to resume re-index for 10 days. This is far beyond it's useful real world
// implications anyway.
private static final int INDEX_TTL = 60 * 60 * 24 * 10;
private static final String MAP_CURSOR_KEY = "cursor";
private static final String MAP_COUNT_KEY = "count";
private static final String MAP_STATUS_KEY = "status";
private static final String MAP_UPDATED_KEY = "lastUpdated";
private final AllApplicationsObservable allApplicationsObservable;
private final IndexLocationStrategyFactory indexLocationStrategyFactory;
private final AllEntityIdsObservable allEntityIdsObservable;
private final IndexProcessorFig indexProcessorFig;
private final MapManager mapManager;
private final AsyncEventService indexService;
private final EntityIndexFactory entityIndexFactory;
@Inject
public ReIndexServiceImpl(
final EntityIndexFactory entityIndexFactory,
final IndexLocationStrategyFactory indexLocationStrategyFactory,
final AllEntityIdsObservable allEntityIdsObservable,
final MapManagerFactory mapManagerFactory,
final AllApplicationsObservable allApplicationsObservable,
final IndexProcessorFig indexProcessorFig,
final AsyncEventService indexService) {
this.entityIndexFactory = entityIndexFactory;
this.indexLocationStrategyFactory = indexLocationStrategyFactory;
this.allEntityIdsObservable = allEntityIdsObservable;
this.allApplicationsObservable = allApplicationsObservable;
this.indexProcessorFig = indexProcessorFig;
this.indexService = indexService;
this.mapManager = mapManagerFactory.createMapManager(RESUME_MAP_SCOPE);
}
@Override
public ReIndexStatus rebuildIndex(final ReIndexRequestBuilder reIndexRequestBuilder) {
// load our last emitted Scope if a cursor is present
final Optional<EdgeScope> cursor = parseCursor(reIndexRequestBuilder.getCursor());
final CursorSeek<Edge> cursorSeek = getResumeEdge(cursor);
final Optional<ApplicationScope> appId = reIndexRequestBuilder.getApplicationScope();
Preconditions.checkArgument(
!(cursor.isPresent() && appId.isPresent()),
"You cannot specify an app id and a cursor. When resuming with cursor you must omit the appid");
final Observable<ApplicationScope> applicationScopes = getApplications(cursor, appId);
final String jobId = StringUtils.sanitizeUUID(UUIDGenerator.newTimeUUID());
final long modifiedSince = reIndexRequestBuilder.getUpdateTimestamp().or(Long.MIN_VALUE);
// create an observable that loads a batch to be indexed
final Observable<List<EdgeScope>> runningReIndex =
allEntityIdsObservable
.getEdgesToEntities(
applicationScopes,
reIndexRequestBuilder.getCollectionName(),
cursorSeek.getSeekValue())
.buffer(indexProcessorFig.getReindexBufferSize())
.doOnNext(
edges -> {
logger.info("Sending batch of {} to be indexed.", edges.size());
indexService.indexBatch(edges, modifiedSince);
});
// start our sampler and state persistence
// take a sample every sample interval to allow us to resume state with minimal loss
// create our flushing collector and flush the edge scopes to it
runningReIndex
.collect(
() -> new FlushingCollector(jobId),
((flushingCollector, edgeScopes) -> flushingCollector.flushBuffer(edgeScopes)))
.doOnNext(flushingCollector -> flushingCollector.complete())
// subscribe on our I/O scheduler and run the task
.subscribeOn(Schedulers.io())
.subscribe(); // want reindex to continually run so leave subscribe.
return new ReIndexStatus(jobId, Status.STARTED, 0, 0);
}
@Override
public ReIndexRequestBuilder getBuilder() {
return new ReIndexRequestBuilderImpl();
}
@Override
public ReIndexStatus getStatus(final String jobId) {
Preconditions.checkNotNull(jobId, "jobId must not be null");
return getIndexResponse(jobId);
}
/**
* Simple collector that counts state, then flushed every time a buffer is provided. Writes final
* state when complete
*/
private class FlushingCollector {
private final String jobId;
private long count;
private FlushingCollector(final String jobId) {
this.jobId = jobId;
}
public void flushBuffer(final List<EdgeScope> buffer) {
count += buffer.size();
// write our cursor state
if (buffer.size() > 0) {
writeCursorState(jobId, buffer.get(buffer.size() - 1));
}
writeStateMeta(jobId, Status.INPROGRESS, count, System.currentTimeMillis());
}
public void complete() {
writeStateMeta(jobId, Status.COMPLETE, count, System.currentTimeMillis());
}
}
/**
* Get the resume edge scope
*
* @param edgeScope The optional edge scope from the cursor
*/
private CursorSeek<Edge> getResumeEdge(final Optional<EdgeScope> edgeScope) {
if (edgeScope.isPresent()) {
return new CursorSeek<>(Optional.of(edgeScope.get().getEdge()));
}
return new CursorSeek<>(Optional.absent());
}
/** Generate an observable for our appliation scope */
private Observable<ApplicationScope> getApplications(
final Optional<EdgeScope> cursor, final Optional<ApplicationScope> appId) {
// cursor is present use it and skip until we hit that app
if (cursor.isPresent()) {
final EdgeScope cursorValue = cursor.get();
// we have a cursor and an application scope that was used.
return allApplicationsObservable
.getData()
.skipWhile(
applicationScope -> !cursorValue.getApplicationScope().equals(applicationScope));
}
// this is intentional. If
else if (appId.isPresent()) {
return Observable.just(appId.get());
}
return allApplicationsObservable
.getData()
.doOnNext(
appScope -> {
// make sure index is initialized on rebuild
entityIndexFactory
.createEntityIndex(
indexLocationStrategyFactory.getIndexLocationStrategy(appScope))
.initialize();
});
}
/** Swap our cursor for an optional edgescope */
private Optional<EdgeScope> parseCursor(final Optional<String> cursor) {
if (!cursor.isPresent()) {
return Optional.absent();
}
// get our cursor
final String persistedCursor = mapManager.getString(cursor.get());
if (persistedCursor == null) {
return Optional.absent();
}
final JsonNode node = CursorSerializerUtil.fromString(persistedCursor);
final EdgeScope edgeScope =
EdgeScopeSerializer.INSTANCE.fromJsonNode(node, CursorSerializerUtil.getMapper());
return Optional.of(edgeScope);
}
/** Write the cursor state to the map in cassandra */
private void writeCursorState(final String jobId, final EdgeScope edge) {
final JsonNode node =
EdgeScopeSerializer.INSTANCE.toNode(CursorSerializerUtil.getMapper(), edge);
final String serializedState = CursorSerializerUtil.asString(node);
mapManager.putString(jobId + MAP_CURSOR_KEY, serializedState, INDEX_TTL);
}
/**
* Write our state meta data into cassandra so everyone can see it
*
* @param jobId
* @param status
* @param processedCount
* @param lastUpdated
*/
private void writeStateMeta(
final String jobId, final Status status, final long processedCount, final long lastUpdated) {
if (logger.isDebugEnabled()) {
logger.debug(
"Flushing state for jobId {}, status {}, processedCount {}, lastUpdated {}",
jobId,
status,
processedCount,
lastUpdated);
}
mapManager.putString(jobId + MAP_STATUS_KEY, status.name());
mapManager.putLong(jobId + MAP_COUNT_KEY, processedCount);
mapManager.putLong(jobId + MAP_UPDATED_KEY, lastUpdated);
}
/**
* Get the index response from the jobId
*
* @param jobId
* @return
*/
private ReIndexStatus getIndexResponse(final String jobId) {
final String stringStatus = mapManager.getString(jobId + MAP_STATUS_KEY);
if (stringStatus == null) {
return new ReIndexStatus(jobId, Status.UNKNOWN, 0, 0);
}
final Status status = Status.valueOf(stringStatus);
final long processedCount = mapManager.getLong(jobId + MAP_COUNT_KEY);
final long lastUpdated = mapManager.getLong(jobId + MAP_COUNT_KEY);
return new ReIndexStatus(jobId, status, processedCount, lastUpdated);
}
}