Ejemplo n.º 1
0
  private void handleIndexOperation(final ElasticsearchIndexEvent elasticsearchIndexEvent)
      throws IndexDocNotFoundException {

    Preconditions.checkNotNull(elasticsearchIndexEvent, "elasticsearchIndexEvent cannot be null");

    final UUID messageId = elasticsearchIndexEvent.getIndexBatchId();
    Preconditions.checkNotNull(messageId, "messageId must not be null");

    final String message = esMapPersistence.getString(messageId.toString());

    final IndexOperationMessage indexOperationMessage;
    if (message == null) {

      // provide some time back pressure before performing a quorum read
      if (queueFig.getQuorumFallback()
          && System.currentTimeMillis()
              > elasticsearchIndexEvent.getCreationTime() + queueFig.getLocalQuorumTimeout()) {

        if (logger.isDebugEnabled()) {
          logger.debug("ES batch with id {} not found, reading with strong consistency", messageId);
        }

        final String highConsistency =
            esMapPersistence.getStringHighConsistency(messageId.toString());
        if (highConsistency == null) {

          throw new RuntimeException(
              "ES batch with id " + messageId + " not found when reading with strong consistency");
        }

        indexOperationMessage =
            ObjectJsonSerializer.INSTANCE.fromString(highConsistency, IndexOperationMessage.class);

      } else {

        throw new IndexDocNotFoundException(elasticsearchIndexEvent.getIndexBatchId());
      }

    } else {

      indexOperationMessage =
          ObjectJsonSerializer.INSTANCE.fromString(message, IndexOperationMessage.class);
    }

    // don't let this continue if there's nothing to index
    if (indexOperationMessage == null || indexOperationMessage.isEmpty()) {
      throw new RuntimeException(
          "IndexOperationMessage cannot be null or empty after retrieving from map persistence");
    }

    // always do a check to ensure the indexes are initialized for the index requests
    initializeEntityIndexes(indexOperationMessage);

    // send it to to be indexed
    indexProducer.put(indexOperationMessage).toBlocking().last();
  }
  /**
   * Get the index response from the jobId
   *
   * @param jobId
   * @return
   */
  private ReIndexStatus getIndexResponse(final String jobId) {

    final String stringStatus = mapManager.getString(jobId + MAP_STATUS_KEY);

    if (stringStatus == null) {
      return new ReIndexStatus(jobId, Status.UNKNOWN, 0, 0);
    }

    final Status status = Status.valueOf(stringStatus);

    final long processedCount = mapManager.getLong(jobId + MAP_COUNT_KEY);
    final long lastUpdated = mapManager.getLong(jobId + MAP_COUNT_KEY);

    return new ReIndexStatus(jobId, status, processedCount, lastUpdated);
  }
Ejemplo n.º 3
0
  /**
   * Queue up an indexOperationMessage for multi region execution
   *
   * @param indexOperationMessage
   */
  public void queueIndexOperationMessage(final IndexOperationMessage indexOperationMessage) {

    // don't try to produce something with nothing
    if (indexOperationMessage == null || indexOperationMessage.isEmpty()) {
      return;
    }

    final String jsonValue = ObjectJsonSerializer.INSTANCE.toString(indexOperationMessage);

    final UUID newMessageId = UUIDGenerator.newTimeUUID();

    final int expirationTimeInSeconds =
        (int) TimeUnit.MILLISECONDS.toSeconds(indexProcessorFig.getIndexMessageTtl());

    // write to the map in ES
    esMapPersistence.putString(newMessageId.toString(), jsonValue, expirationTimeInSeconds);

    // now queue up the index message

    final ElasticsearchIndexEvent elasticsearchIndexEvent =
        new ElasticsearchIndexEvent(queueFig.getPrimaryRegion(), newMessageId);

    // send to the topic so all regions index the batch

    offerTopic(elasticsearchIndexEvent);
  }
  /**
   * Write our state meta data into cassandra so everyone can see it
   *
   * @param jobId
   * @param status
   * @param processedCount
   * @param lastUpdated
   */
  private void writeStateMeta(
      final String jobId, final Status status, final long processedCount, final long lastUpdated) {

    if (logger.isDebugEnabled()) {
      logger.debug(
          "Flushing state for jobId {}, status {}, processedCount {}, lastUpdated {}",
          jobId,
          status,
          processedCount,
          lastUpdated);
    }

    mapManager.putString(jobId + MAP_STATUS_KEY, status.name());
    mapManager.putLong(jobId + MAP_COUNT_KEY, processedCount);
    mapManager.putLong(jobId + MAP_UPDATED_KEY, lastUpdated);
  }
  /** Write the cursor state to the map in cassandra */
  private void writeCursorState(final String jobId, final EdgeScope edge) {

    final JsonNode node =
        EdgeScopeSerializer.INSTANCE.toNode(CursorSerializerUtil.getMapper(), edge);

    final String serializedState = CursorSerializerUtil.asString(node);

    mapManager.putString(jobId + MAP_CURSOR_KEY, serializedState, INDEX_TTL);
  }
Ejemplo n.º 6
0
  /**
   * 4. Delete all entity documents out of elasticsearch. 5. Compact Graph so that it deletes the
   * marked values. 6. Delete entity from cassandra using the map manager.
   */
  private Id deleteAsync(MapManager mapManager, ApplicationScope applicationScope, Id entityId) {
    try {
      // Step 4 && 5

      if (!skipIndexingForType(entityId.getType(), applicationScope)) {

        asyncEventService.queueEntityDelete(applicationScope, entityId);
      }
      // Step 6
      // delete from our UUID index
      mapManager.delete(entityId.getUuid().toString());
      return entityId;
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }
  /** Swap our cursor for an optional edgescope */
  private Optional<EdgeScope> parseCursor(final Optional<String> cursor) {

    if (!cursor.isPresent()) {
      return Optional.absent();
    }

    // get our cursor
    final String persistedCursor = mapManager.getString(cursor.get());

    if (persistedCursor == null) {
      return Optional.absent();
    }

    final JsonNode node = CursorSerializerUtil.fromString(persistedCursor);

    final EdgeScope edgeScope =
        EdgeScopeSerializer.INSTANCE.fromJsonNode(node, CursorSerializerUtil.getMapper());

    return Optional.of(edgeScope);
  }