Beispiel #1
0
 @Override
 protected List<Metacard> query(CatalogFacade framework, int startIndex, Filter filter) {
   QueryImpl query = new QueryImpl(filter);
   query.setRequestsTotalResultsCount(false);
   query.setPageSize(batchSize);
   query.setSortBy(new SortByImpl(Metacard.MODIFIED, SortOrder.DESCENDING));
   QueryRequest queryRequest = new QueryRequestImpl(query);
   query.setStartIndex(startIndex);
   SourceResponse response;
   try {
     LOGGER.debug("Querying with startIndex: {}", startIndex);
     response = framework.query(queryRequest);
   } catch (UnsupportedQueryException e) {
     printErrorMessage(String.format("Received error from Framework: %s%n", e.getMessage()));
     return null;
   } catch (SourceUnavailableException e) {
     printErrorMessage(String.format("Received error from Frameworks: %s%n", e.getMessage()));
     return null;
   } catch (FederationException e) {
     printErrorMessage(String.format("Received error from Frameworks: %s%n", e.getMessage()));
     return null;
   }
   if (response.getProcessingDetails() != null && !response.getProcessingDetails().isEmpty()) {
     for (SourceProcessingDetails details : response.getProcessingDetails()) {
       LOGGER.debug("Got Issues: {}", details.getWarnings());
     }
     return null;
   }
   return response.getResults().stream().map(Result::getMetacard).collect(Collectors.toList());
 }
  @Override
  protected Object doExecute() throws Exception {

    String formatString =
        "%1$-33s %2$-26s %3$-" + TITLE_MAX_LENGTH + "s %4$-" + EXCERPT_MAX_LENGTH + "s%n";

    CatalogFacade catalogProvider = getCatalog();

    Filter filter = null;
    if (cqlFilter != null) {
      filter = CQL.toFilter(cqlFilter);
    } else {
      if (searchPhrase == null) {
        searchPhrase = "*";
      }
      if (caseSensitive) {
        filter =
            getFilterBuilder()
                .attribute(Metacard.ANY_TEXT)
                .is()
                .like()
                .caseSensitiveText(searchPhrase);
      } else {
        filter = getFilterBuilder().attribute(Metacard.ANY_TEXT).is().like().text(searchPhrase);
      }
    }

    QueryImpl query = new QueryImpl(filter);

    query.setRequestsTotalResultsCount(true);

    if (numberOfItems > -1) {
      query.setPageSize(numberOfItems);
    }

    long start = System.currentTimeMillis();

    SourceResponse response = catalogProvider.query(new QueryRequestImpl(query));

    long end = System.currentTimeMillis();

    int size = 0;
    if (response.getResults() != null) {
      size = response.getResults().size();
    }

    console.println();
    console.printf(
        " %d result(s) out of %s%d%s in %3.3f seconds",
        (size),
        Ansi.ansi().fg(Ansi.Color.CYAN).toString(),
        response.getHits(),
        Ansi.ansi().reset().toString(),
        (end - start) / MILLISECONDS_PER_SECOND);
    console.printf(formatString, "", "", "", "");
    printHeaderMessage(String.format(formatString, ID, DATE, TITLE, EXCERPT));

    for (Result result : response.getResults()) {
      Metacard metacard = result.getMetacard();

      String title = (metacard.getTitle() != null ? metacard.getTitle() : "N/A");
      String excerpt = "N/A";
      String modifiedDate = "";

      if (searchPhrase != null) {
        if (metacard.getMetadata() != null) {
          XPathHelper helper = new XPathHelper(metacard.getMetadata());
          String indexedText = helper.getDocument().getDocumentElement().getTextContent();
          indexedText = indexedText.replaceAll("\\r\\n|\\r|\\n", " ");

          String normalizedSearchPhrase = searchPhrase.replaceAll("\\*", "");

          int index = -1;

          if (caseSensitive) {
            index = indexedText.indexOf(normalizedSearchPhrase);
          } else {
            index = indexedText.toLowerCase().indexOf(normalizedSearchPhrase.toLowerCase());
          }

          if (index != -1) {
            int contextLength = (EXCERPT_MAX_LENGTH - normalizedSearchPhrase.length() - 8) / 2;
            excerpt = "..." + indexedText.substring(Math.max(index - contextLength, 0), index);
            excerpt = excerpt + Ansi.ansi().fg(Ansi.Color.GREEN).toString();
            excerpt =
                excerpt + indexedText.substring(index, index + normalizedSearchPhrase.length());
            excerpt = excerpt + Ansi.ansi().reset().toString();
            excerpt =
                excerpt
                    + indexedText.substring(
                        index + normalizedSearchPhrase.length(),
                        Math.min(
                            indexedText.length(),
                            index + normalizedSearchPhrase.length() + contextLength))
                    + "...";
          }
        }
      }

      if (metacard.getModifiedDate() != null) {
        modifiedDate =
            new DateTime(metacard.getModifiedDate().getTime()).toString(DATETIME_FORMATTER);
      }

      console.printf(
          formatString,
          metacard.getId(),
          modifiedDate,
          title.substring(0, Math.min(title.length(), TITLE_MAX_LENGTH)),
          excerpt);
    }

    return null;
  }
Beispiel #3
0
  @Override
  protected Object executeWithSubject() throws Exception {

    List<CatalogProvider> providers = getCatalogProviders();

    if (providers.isEmpty() || providers.size() < 2) {
      console.println("Not enough CatalogProviders installed to migrate");
      return null;
    }

    console.println("The \"FROM\" provider is: " + providers.get(0).getClass().getSimpleName());
    CatalogProvider provider = providers.get(1);
    console.println("The \"TO\" provider is: " + provider.getClass().getSimpleName());
    String answer = getInput("Do you wish to continue? (yes/no)");
    if (!"yes".equalsIgnoreCase(answer)) {
      console.println();
      console.println("Now exiting...");
      console.flush();
      return null;
    }

    ingestProvider = new Provider(provider);

    framework = getCatalog();

    start = System.currentTimeMillis();

    final Filter filter =
        (cqlFilter != null)
            ? CQL.toFilter(cqlFilter)
            : getFilter(getFilterStartTime(start), start, Metacard.MODIFIED);

    QueryImpl query = new QueryImpl(filter);
    query.setRequestsTotalResultsCount(true);
    query.setPageSize(batchSize);
    query.setSortBy(new SortByImpl(Metacard.MODIFIED, SortOrder.DESCENDING));
    QueryRequest queryRequest = new QueryRequestImpl(query);
    SourceResponse response;
    try {
      response = framework.query(queryRequest);
    } catch (FederationException e) {
      printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
      return null;
    } catch (SourceUnavailableException e) {
      printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
      return null;
    } catch (UnsupportedQueryException e) {
      printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
      return null;
    }
    final long totalHits = response.getHits();
    final long totalPossible;
    if (totalHits == 0) {
      console.println("No records were found to replicate.");
      return null;
    }

    // If the maxMetacards is set, restrict the totalPossible to the number of maxMetacards
    if (maxMetacards > 0 && maxMetacards <= totalHits) {
      totalPossible = maxMetacards;
    } else {
      totalPossible = totalHits;
    }

    console.println("Starting migration for " + totalPossible + " Records");

    if (multithreaded > 1 && totalPossible > batchSize) {
      BlockingQueue<Runnable> blockingQueue = new ArrayBlockingQueue<Runnable>(multithreaded);
      RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy();
      final ExecutorService executorService =
          new ThreadPoolExecutor(
              multithreaded,
              multithreaded,
              0L,
              TimeUnit.MILLISECONDS,
              blockingQueue,
              rejectedExecutionHandler);
      console.printf("Running %d threads during replication.%n", multithreaded);

      do {
        LOGGER.debug("In loop at iteration {}", queryIndex.get());
        executorService.submit(
            () -> {
              int count = queryAndIngest(framework, ingestProvider, queryIndex.get(), filter);
              printProgressAndFlush(start, totalPossible, ingestCount.addAndGet(count));
            });
      } while (queryIndex.addAndGet(batchSize) <= totalPossible);
      executorService.shutdown();

      while (!executorService.isTerminated()) {
        try {
          TimeUnit.SECONDS.sleep(1);
        } catch (InterruptedException e) {
          // ignore
        }
      }
    } else {
      do {
        int count = queryAndIngest(framework, ingestProvider, queryIndex.get(), filter);
        printProgressAndFlush(start, totalPossible, ingestCount.addAndGet(count));
      } while (queryIndex.addAndGet(batchSize) <= totalPossible);
    }

    console.println();
    long end = System.currentTimeMillis();
    String completed =
        String.format(
            " %d record(s) replicated; %d record(s) failed; completed in %3.3f seconds.",
            ingestCount.get(), failedCount.get(), (end - start) / MS_PER_SECOND);
    LOGGER.info("Replication Complete: {}", completed);
    console.println(completed);

    return null;
  }