/** * this method will call initialize for each message, since we are caching the entity indexes, we * don't worry about aggregating by app id * * @param indexOperationMessage */ private void initializeEntityIndexes(final IndexOperationMessage indexOperationMessage) { // create a set so we can have a unique list of appIds for which we call createEntityIndex Set<UUID> appIds = new HashSet<>(); // loop through all indexRequests and add the appIds to the set indexOperationMessage .getIndexRequests() .forEach( req -> { UUID appId = IndexingUtils.getApplicationIdFromIndexDocId(req.documentId); appIds.add(appId); }); // loop through all deindexRequests and add the appIds to the set indexOperationMessage .getDeIndexRequests() .forEach( req -> { UUID appId = IndexingUtils.getApplicationIdFromIndexDocId(req.documentId); appIds.add(appId); }); // for each of the appIds in the unique set, call create entity index to ensure the aliases are // created appIds.forEach( appId -> { ApplicationScope appScope = CpNamingUtils.getApplicationScope(appId); entityIndexFactory.createEntityIndex( indexLocationStrategyFactory.getIndexLocationStrategy(appScope)); }); }
private void handleIndexOperation(final ElasticsearchIndexEvent elasticsearchIndexEvent) throws IndexDocNotFoundException { Preconditions.checkNotNull(elasticsearchIndexEvent, "elasticsearchIndexEvent cannot be null"); final UUID messageId = elasticsearchIndexEvent.getIndexBatchId(); Preconditions.checkNotNull(messageId, "messageId must not be null"); final String message = esMapPersistence.getString(messageId.toString()); final IndexOperationMessage indexOperationMessage; if (message == null) { // provide some time back pressure before performing a quorum read if (queueFig.getQuorumFallback() && System.currentTimeMillis() > elasticsearchIndexEvent.getCreationTime() + queueFig.getLocalQuorumTimeout()) { if (logger.isDebugEnabled()) { logger.debug("ES batch with id {} not found, reading with strong consistency", messageId); } final String highConsistency = esMapPersistence.getStringHighConsistency(messageId.toString()); if (highConsistency == null) { throw new RuntimeException( "ES batch with id " + messageId + " not found when reading with strong consistency"); } indexOperationMessage = ObjectJsonSerializer.INSTANCE.fromString(highConsistency, IndexOperationMessage.class); } else { throw new IndexDocNotFoundException(elasticsearchIndexEvent.getIndexBatchId()); } } else { indexOperationMessage = ObjectJsonSerializer.INSTANCE.fromString(message, IndexOperationMessage.class); } // don't let this continue if there's nothing to index if (indexOperationMessage == null || indexOperationMessage.isEmpty()) { throw new RuntimeException( "IndexOperationMessage cannot be null or empty after retrieving from map persistence"); } // always do a check to ensure the indexes are initialized for the index requests initializeEntityIndexes(indexOperationMessage); // send it to to be indexed indexProducer.put(indexOperationMessage).toBlocking().last(); }
/** * Queue up an indexOperationMessage for multi region execution * * @param indexOperationMessage */ public void queueIndexOperationMessage(final IndexOperationMessage indexOperationMessage) { // don't try to produce something with nothing if (indexOperationMessage == null || indexOperationMessage.isEmpty()) { return; } final String jsonValue = ObjectJsonSerializer.INSTANCE.toString(indexOperationMessage); final UUID newMessageId = UUIDGenerator.newTimeUUID(); final int expirationTimeInSeconds = (int) TimeUnit.MILLISECONDS.toSeconds(indexProcessorFig.getIndexMessageTtl()); // write to the map in ES esMapPersistence.putString(newMessageId.toString(), jsonValue, expirationTimeInSeconds); // now queue up the index message final ElasticsearchIndexEvent elasticsearchIndexEvent = new ElasticsearchIndexEvent(queueFig.getPrimaryRegion(), newMessageId); // send to the topic so all regions index the batch offerTopic(elasticsearchIndexEvent); }
public void indexBatch(final List<EdgeScope> edges, final long updatedSince) { IndexOperationMessage batch = new IndexOperationMessage(); for (EdgeScope e : edges) { EntityIndexOperation entityIndexOperation = new EntityIndexOperation( e.getApplicationScope(), e.getEdge().getTargetNode(), updatedSince); IndexOperationMessage indexOperationMessage = eventBuilder.buildEntityIndex(entityIndexOperation).toBlocking().lastOrDefault(null); if (indexOperationMessage != null) { batch.ingest(indexOperationMessage); } } queueIndexOperationMessage(batch); }
/** Submit results to index and return the queue messages to be ack'd */ private List<QueueMessage> submitToIndex(List<IndexEventResult> indexEventResults) { // if nothing came back then return empty list if (indexEventResults == null) { return new ArrayList<>(0); } IndexOperationMessage combined = new IndexOperationMessage(); List<QueueMessage> queueMessages = indexEventResults .stream() // filter out messages that are not present, they were not processed and put into the // results .filter(result -> result.getQueueMessage().isPresent()) .map( indexEventResult -> { // record the cycle time messageCycle.update( System.currentTimeMillis() - indexEventResult.getCreationTime()); // ingest each index op into our combined, single index op for the index producer if (indexEventResult.getIndexOperationMessage().isPresent()) { combined.ingest(indexEventResult.getIndexOperationMessage().get()); } return indexEventResult.getQueueMessage().get(); }) // collect into a list of QueueMessages that can be ack'd later .collect(Collectors.toList()); queueIndexOperationMessage(combined); return queueMessages; }
/** * calls the event handlers and returns a result with information on whether it needs to be ack'd * and whether it needs to be indexed * * @param messages * @return */ private List<IndexEventResult> callEventHandlers(final List<QueueMessage> messages) { if (logger.isDebugEnabled()) { logger.debug("callEventHandlers with {} message(s)", messages.size()); } Stream<IndexEventResult> indexEventResults = messages .stream() .map( message -> { if (logger.isDebugEnabled()) { logger.debug( "Queue message with ID {} has been received {} time(s)", message.getMessageId(), message.getReceiveCount()); } AsyncEvent event = null; try { event = (AsyncEvent) message.getBody(); } catch (ClassCastException cce) { logger.error("Failed to deserialize message body", cce); return new IndexEventResult( Optional.absent(), Optional.absent(), System.currentTimeMillis()); } if (event == null) { logger.error("AsyncEvent type or event is null!"); return new IndexEventResult( Optional.absent(), Optional.absent(), System.currentTimeMillis()); } final AsyncEvent thisEvent = event; if (logger.isDebugEnabled()) { logger.debug("Processing event with type {}", event.getClass().getSimpleName()); } try { IndexOperationMessage single = new IndexOperationMessage(); // normal indexing event for an entity if (event instanceof EntityIndexEvent) { single = handleEntityIndexUpdate(message); } // normal indexing event for an edge else if (event instanceof EdgeIndexEvent) { single = handleEdgeIndex(message); } // deletes are 2-part, actual IO to delete data, then queue up a de-index else if (event instanceof EdgeDeleteEvent) { single = handleEdgeDelete(message); } // deletes are 2-part, actual IO to delete data, then queue up a de-index else if (event instanceof EntityDeleteEvent) { single = handleEntityDelete(message); } // initialization has special logic, therefore a special event type and no index // operation message else if (event instanceof InitializeApplicationIndexEvent) { handleInitializeApplicationIndex(event, message); } // this is the main event that pulls the index doc from map persistence and // hands to the index producer else if (event instanceof ElasticsearchIndexEvent) { handleIndexOperation((ElasticsearchIndexEvent) event); } else if (event instanceof DeIndexOldVersionsEvent) { single = handleDeIndexOldVersionEvent((DeIndexOldVersionsEvent) event); } else { throw new Exception( "Unknown EventType for message: " + message.getStringBody().trim()); } if (!(event instanceof ElasticsearchIndexEvent) && !(event instanceof InitializeApplicationIndexEvent) && single.isEmpty()) { logger.warn( "No index operation messages came back from event processing for msg: {} ", message.getStringBody().trim()); } // if no exception happens and the QueueMessage is returned in these results, it // will get ack'd return new IndexEventResult( Optional.of(single), Optional.of(message), thisEvent.getCreationTime()); } catch (IndexDocNotFoundException e) { // this exception is throw when we wait before trying quorum read on map // persistence. // return empty event result so the event's message doesn't get ack'd if (logger.isDebugEnabled()) { logger.debug(e.getMessage()); } return new IndexEventResult( Optional.absent(), Optional.absent(), thisEvent.getCreationTime()); } catch (Exception e) { // NPEs don't have a detail message, so add something for our log statement to // identify better final String errorMessage; if (e instanceof NullPointerException) { errorMessage = "NullPointerException"; } else { errorMessage = e.getMessage(); } // if the event fails to process, log and return empty message result so it // doesn't get ack'd logger.error( "{}. Failed to process message: {}", errorMessage, message.getStringBody().trim()); return new IndexEventResult( Optional.absent(), Optional.absent(), thisEvent.getCreationTime()); } }); return indexEventResults.collect(Collectors.toList()); }