private void handleIndexOperation(final ElasticsearchIndexEvent elasticsearchIndexEvent)
throws IndexDocNotFoundException {
Preconditions.checkNotNull(elasticsearchIndexEvent, "elasticsearchIndexEvent cannot be null");
final UUID messageId = elasticsearchIndexEvent.getIndexBatchId();
Preconditions.checkNotNull(messageId, "messageId must not be null");
final String message = esMapPersistence.getString(messageId.toString());
final IndexOperationMessage indexOperationMessage;
if (message == null) {
// provide some time back pressure before performing a quorum read
if (queueFig.getQuorumFallback()
&& System.currentTimeMillis()
> elasticsearchIndexEvent.getCreationTime() + queueFig.getLocalQuorumTimeout()) {
if (logger.isDebugEnabled()) {
logger.debug("ES batch with id {} not found, reading with strong consistency", messageId);
}
final String highConsistency =
esMapPersistence.getStringHighConsistency(messageId.toString());
if (highConsistency == null) {
throw new RuntimeException(
"ES batch with id " + messageId + " not found when reading with strong consistency");
}
indexOperationMessage =
ObjectJsonSerializer.INSTANCE.fromString(highConsistency, IndexOperationMessage.class);
} else {
throw new IndexDocNotFoundException(elasticsearchIndexEvent.getIndexBatchId());
}
} else {
indexOperationMessage =
ObjectJsonSerializer.INSTANCE.fromString(message, IndexOperationMessage.class);
}
// don't let this continue if there's nothing to index
if (indexOperationMessage == null || indexOperationMessage.isEmpty()) {
throw new RuntimeException(
"IndexOperationMessage cannot be null or empty after retrieving from map persistence");
}
// always do a check to ensure the indexes are initialized for the index requests
initializeEntityIndexes(indexOperationMessage);
// send it to to be indexed
indexProducer.put(indexOperationMessage).toBlocking().last();
}
/**
* Get the index response from the jobId
*
* @param jobId
* @return
*/
private ReIndexStatus getIndexResponse(final String jobId) {
final String stringStatus = mapManager.getString(jobId + MAP_STATUS_KEY);
if (stringStatus == null) {
return new ReIndexStatus(jobId, Status.UNKNOWN, 0, 0);
}
final Status status = Status.valueOf(stringStatus);
final long processedCount = mapManager.getLong(jobId + MAP_COUNT_KEY);
final long lastUpdated = mapManager.getLong(jobId + MAP_COUNT_KEY);
return new ReIndexStatus(jobId, status, processedCount, lastUpdated);
}
/**
* Queue up an indexOperationMessage for multi region execution
*
* @param indexOperationMessage
*/
public void queueIndexOperationMessage(final IndexOperationMessage indexOperationMessage) {
// don't try to produce something with nothing
if (indexOperationMessage == null || indexOperationMessage.isEmpty()) {
return;
}
final String jsonValue = ObjectJsonSerializer.INSTANCE.toString(indexOperationMessage);
final UUID newMessageId = UUIDGenerator.newTimeUUID();
final int expirationTimeInSeconds =
(int) TimeUnit.MILLISECONDS.toSeconds(indexProcessorFig.getIndexMessageTtl());
// write to the map in ES
esMapPersistence.putString(newMessageId.toString(), jsonValue, expirationTimeInSeconds);
// now queue up the index message
final ElasticsearchIndexEvent elasticsearchIndexEvent =
new ElasticsearchIndexEvent(queueFig.getPrimaryRegion(), newMessageId);
// send to the topic so all regions index the batch
offerTopic(elasticsearchIndexEvent);
}
/**
* Write our state meta data into cassandra so everyone can see it
*
* @param jobId
* @param status
* @param processedCount
* @param lastUpdated
*/
private void writeStateMeta(
final String jobId, final Status status, final long processedCount, final long lastUpdated) {
if (logger.isDebugEnabled()) {
logger.debug(
"Flushing state for jobId {}, status {}, processedCount {}, lastUpdated {}",
jobId,
status,
processedCount,
lastUpdated);
}
mapManager.putString(jobId + MAP_STATUS_KEY, status.name());
mapManager.putLong(jobId + MAP_COUNT_KEY, processedCount);
mapManager.putLong(jobId + MAP_UPDATED_KEY, lastUpdated);
}
/** Write the cursor state to the map in cassandra */
private void writeCursorState(final String jobId, final EdgeScope edge) {
final JsonNode node =
EdgeScopeSerializer.INSTANCE.toNode(CursorSerializerUtil.getMapper(), edge);
final String serializedState = CursorSerializerUtil.asString(node);
mapManager.putString(jobId + MAP_CURSOR_KEY, serializedState, INDEX_TTL);
}
/**
* 4. Delete all entity documents out of elasticsearch. 5. Compact Graph so that it deletes the
* marked values. 6. Delete entity from cassandra using the map manager.
*/
private Id deleteAsync(MapManager mapManager, ApplicationScope applicationScope, Id entityId) {
try {
// Step 4 && 5
if (!skipIndexingForType(entityId.getType(), applicationScope)) {
asyncEventService.queueEntityDelete(applicationScope, entityId);
}
// Step 6
// delete from our UUID index
mapManager.delete(entityId.getUuid().toString());
return entityId;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/** Swap our cursor for an optional edgescope */
private Optional<EdgeScope> parseCursor(final Optional<String> cursor) {
if (!cursor.isPresent()) {
return Optional.absent();
}
// get our cursor
final String persistedCursor = mapManager.getString(cursor.get());
if (persistedCursor == null) {
return Optional.absent();
}
final JsonNode node = CursorSerializerUtil.fromString(persistedCursor);
final EdgeScope edgeScope =
EdgeScopeSerializer.INSTANCE.fromJsonNode(node, CursorSerializerUtil.getMapper());
return Optional.of(edgeScope);
}