Пример #1
0
  /**
   * this method will call initialize for each message, since we are caching the entity indexes, we
   * don't worry about aggregating by app id
   *
   * @param indexOperationMessage
   */
  private void initializeEntityIndexes(final IndexOperationMessage indexOperationMessage) {

    // create a set so we can have a unique list of appIds for which we call createEntityIndex
    Set<UUID> appIds = new HashSet<>();

    // loop through all indexRequests and add the appIds to the set
    indexOperationMessage
        .getIndexRequests()
        .forEach(
            req -> {
              UUID appId = IndexingUtils.getApplicationIdFromIndexDocId(req.documentId);
              appIds.add(appId);
            });

    // loop through all deindexRequests and add the appIds to the set
    indexOperationMessage
        .getDeIndexRequests()
        .forEach(
            req -> {
              UUID appId = IndexingUtils.getApplicationIdFromIndexDocId(req.documentId);
              appIds.add(appId);
            });

    // for each of the appIds in the unique set, call create entity index to ensure the aliases are
    // created
    appIds.forEach(
        appId -> {
          ApplicationScope appScope = CpNamingUtils.getApplicationScope(appId);
          entityIndexFactory.createEntityIndex(
              indexLocationStrategyFactory.getIndexLocationStrategy(appScope));
        });
  }
Пример #2
0
  private void handleIndexOperation(final ElasticsearchIndexEvent elasticsearchIndexEvent)
      throws IndexDocNotFoundException {

    Preconditions.checkNotNull(elasticsearchIndexEvent, "elasticsearchIndexEvent cannot be null");

    final UUID messageId = elasticsearchIndexEvent.getIndexBatchId();
    Preconditions.checkNotNull(messageId, "messageId must not be null");

    final String message = esMapPersistence.getString(messageId.toString());

    final IndexOperationMessage indexOperationMessage;
    if (message == null) {

      // provide some time back pressure before performing a quorum read
      if (queueFig.getQuorumFallback()
          && System.currentTimeMillis()
              > elasticsearchIndexEvent.getCreationTime() + queueFig.getLocalQuorumTimeout()) {

        if (logger.isDebugEnabled()) {
          logger.debug("ES batch with id {} not found, reading with strong consistency", messageId);
        }

        final String highConsistency =
            esMapPersistence.getStringHighConsistency(messageId.toString());
        if (highConsistency == null) {

          throw new RuntimeException(
              "ES batch with id " + messageId + " not found when reading with strong consistency");
        }

        indexOperationMessage =
            ObjectJsonSerializer.INSTANCE.fromString(highConsistency, IndexOperationMessage.class);

      } else {

        throw new IndexDocNotFoundException(elasticsearchIndexEvent.getIndexBatchId());
      }

    } else {

      indexOperationMessage =
          ObjectJsonSerializer.INSTANCE.fromString(message, IndexOperationMessage.class);
    }

    // don't let this continue if there's nothing to index
    if (indexOperationMessage == null || indexOperationMessage.isEmpty()) {
      throw new RuntimeException(
          "IndexOperationMessage cannot be null or empty after retrieving from map persistence");
    }

    // always do a check to ensure the indexes are initialized for the index requests
    initializeEntityIndexes(indexOperationMessage);

    // send it to to be indexed
    indexProducer.put(indexOperationMessage).toBlocking().last();
  }
Пример #3
0
  /**
   * Queue up an indexOperationMessage for multi region execution
   *
   * @param indexOperationMessage
   */
  public void queueIndexOperationMessage(final IndexOperationMessage indexOperationMessage) {

    // don't try to produce something with nothing
    if (indexOperationMessage == null || indexOperationMessage.isEmpty()) {
      return;
    }

    final String jsonValue = ObjectJsonSerializer.INSTANCE.toString(indexOperationMessage);

    final UUID newMessageId = UUIDGenerator.newTimeUUID();

    final int expirationTimeInSeconds =
        (int) TimeUnit.MILLISECONDS.toSeconds(indexProcessorFig.getIndexMessageTtl());

    // write to the map in ES
    esMapPersistence.putString(newMessageId.toString(), jsonValue, expirationTimeInSeconds);

    // now queue up the index message

    final ElasticsearchIndexEvent elasticsearchIndexEvent =
        new ElasticsearchIndexEvent(queueFig.getPrimaryRegion(), newMessageId);

    // send to the topic so all regions index the batch

    offerTopic(elasticsearchIndexEvent);
  }
Пример #4
0
  public void indexBatch(final List<EdgeScope> edges, final long updatedSince) {

    IndexOperationMessage batch = new IndexOperationMessage();

    for (EdgeScope e : edges) {

      EntityIndexOperation entityIndexOperation =
          new EntityIndexOperation(
              e.getApplicationScope(), e.getEdge().getTargetNode(), updatedSince);

      IndexOperationMessage indexOperationMessage =
          eventBuilder.buildEntityIndex(entityIndexOperation).toBlocking().lastOrDefault(null);

      if (indexOperationMessage != null) {
        batch.ingest(indexOperationMessage);
      }
    }

    queueIndexOperationMessage(batch);
  }
Пример #5
0
  /** Submit results to index and return the queue messages to be ack'd */
  private List<QueueMessage> submitToIndex(List<IndexEventResult> indexEventResults) {

    // if nothing came back then return empty list
    if (indexEventResults == null) {
      return new ArrayList<>(0);
    }

    IndexOperationMessage combined = new IndexOperationMessage();
    List<QueueMessage> queueMessages =
        indexEventResults
            .stream()

            // filter out messages that are not present, they were not processed and put into the
            // results
            .filter(result -> result.getQueueMessage().isPresent())
            .map(
                indexEventResult -> {

                  // record the cycle time
                  messageCycle.update(
                      System.currentTimeMillis() - indexEventResult.getCreationTime());

                  // ingest each index op into our combined, single index op for the index producer
                  if (indexEventResult.getIndexOperationMessage().isPresent()) {
                    combined.ingest(indexEventResult.getIndexOperationMessage().get());
                  }

                  return indexEventResult.getQueueMessage().get();
                })
            // collect into a list of QueueMessages that can be ack'd later
            .collect(Collectors.toList());

    queueIndexOperationMessage(combined);

    return queueMessages;
  }
Пример #6
0
  /**
   * calls the event handlers and returns a result with information on whether it needs to be ack'd
   * and whether it needs to be indexed
   *
   * @param messages
   * @return
   */
  private List<IndexEventResult> callEventHandlers(final List<QueueMessage> messages) {

    if (logger.isDebugEnabled()) {
      logger.debug("callEventHandlers with {} message(s)", messages.size());
    }

    Stream<IndexEventResult> indexEventResults =
        messages
            .stream()
            .map(
                message -> {
                  if (logger.isDebugEnabled()) {
                    logger.debug(
                        "Queue message with ID {} has been received {} time(s)",
                        message.getMessageId(),
                        message.getReceiveCount());
                  }

                  AsyncEvent event = null;
                  try {
                    event = (AsyncEvent) message.getBody();

                  } catch (ClassCastException cce) {
                    logger.error("Failed to deserialize message body", cce);
                    return new IndexEventResult(
                        Optional.absent(), Optional.absent(), System.currentTimeMillis());
                  }

                  if (event == null) {
                    logger.error("AsyncEvent type or event is null!");
                    return new IndexEventResult(
                        Optional.absent(), Optional.absent(), System.currentTimeMillis());
                  }

                  final AsyncEvent thisEvent = event;

                  if (logger.isDebugEnabled()) {
                    logger.debug("Processing event with type {}", event.getClass().getSimpleName());
                  }

                  try {

                    IndexOperationMessage single = new IndexOperationMessage();

                    // normal indexing event for an entity
                    if (event instanceof EntityIndexEvent) {

                      single = handleEntityIndexUpdate(message);

                    }
                    // normal indexing event for an edge
                    else if (event instanceof EdgeIndexEvent) {

                      single = handleEdgeIndex(message);

                    }
                    // deletes are 2-part, actual IO to delete data, then queue up a de-index
                    else if (event instanceof EdgeDeleteEvent) {

                      single = handleEdgeDelete(message);
                    }
                    // deletes are 2-part, actual IO to delete data, then queue up a de-index
                    else if (event instanceof EntityDeleteEvent) {

                      single = handleEntityDelete(message);
                    }
                    // initialization has special logic, therefore a special event type and no index
                    // operation message
                    else if (event instanceof InitializeApplicationIndexEvent) {

                      handleInitializeApplicationIndex(event, message);
                    }
                    // this is the main event that pulls the index doc from map persistence and
                    // hands to the index producer
                    else if (event instanceof ElasticsearchIndexEvent) {

                      handleIndexOperation((ElasticsearchIndexEvent) event);

                    } else if (event instanceof DeIndexOldVersionsEvent) {

                      single = handleDeIndexOldVersionEvent((DeIndexOldVersionsEvent) event);

                    } else {

                      throw new Exception(
                          "Unknown EventType for message: " + message.getStringBody().trim());
                    }

                    if (!(event instanceof ElasticsearchIndexEvent)
                        && !(event instanceof InitializeApplicationIndexEvent)
                        && single.isEmpty()) {
                      logger.warn(
                          "No index operation messages came back from event processing for msg: {} ",
                          message.getStringBody().trim());
                    }

                    // if no exception happens and the QueueMessage is returned in these results, it
                    // will get ack'd
                    return new IndexEventResult(
                        Optional.of(single), Optional.of(message), thisEvent.getCreationTime());

                  } catch (IndexDocNotFoundException e) {

                    // this exception is throw when we wait before trying quorum read on map
                    // persistence.
                    // return empty event result so the event's message doesn't get ack'd
                    if (logger.isDebugEnabled()) {
                      logger.debug(e.getMessage());
                    }
                    return new IndexEventResult(
                        Optional.absent(), Optional.absent(), thisEvent.getCreationTime());

                  } catch (Exception e) {

                    // NPEs don't have a detail message, so add something for our log statement to
                    // identify better
                    final String errorMessage;
                    if (e instanceof NullPointerException) {
                      errorMessage = "NullPointerException";
                    } else {
                      errorMessage = e.getMessage();
                    }

                    // if the event fails to process, log and return empty message result so it
                    // doesn't get ack'd
                    logger.error(
                        "{}. Failed to process message: {}",
                        errorMessage,
                        message.getStringBody().trim());
                    return new IndexEventResult(
                        Optional.absent(), Optional.absent(), thisEvent.getCreationTime());
                  }
                });

    return indexEventResults.collect(Collectors.toList());
  }