Exemple #1
0
  public CatalogProvider waitForCatalogProvider() throws InterruptedException {
    LOGGER.info("Waiting for CatalogProvider to become available.");
    serviceManager.printInactiveBundles();

    CatalogProvider provider = null;
    long timeoutLimit = System.currentTimeMillis() + CATALOG_PROVIDER_TIMEOUT;
    boolean available = false;

    while (!available) {
      if (provider == null) {
        ServiceReference<CatalogProvider> providerRef =
            serviceManager.getServiceReference(CatalogProvider.class);
        if (providerRef != null) {
          provider = serviceManager.getService(providerRef);
        }
      }

      if (provider != null) {
        available = provider.isAvailable();
      }

      if (!available) {
        if (System.currentTimeMillis() > timeoutLimit) {
          LOGGER.info("CatalogProvider.isAvailable = false");
          serviceManager.printInactiveBundles();
          fail(
              String.format(
                  "Catalog provider timed out after %d minutes.",
                  TimeUnit.MILLISECONDS.toMinutes(CATALOG_PROVIDER_TIMEOUT)));
        }
        Thread.sleep(100);
      }
    }

    LOGGER.info("CatalogProvider is available.");
    return provider;
  }
  /** ********************************************************************************* */
  private SourceMetricsImpl configureSourceMetrics(String sourceId) throws Exception {

    catalogProvider = mock(CatalogProvider.class);
    when(catalogProvider.getId()).thenReturn(sourceId);

    fedSource = mock(FederatedSource.class);
    when(fedSource.getId()).thenReturn("fs-1");

    sourceMetrics = new SourceMetricsImpl();
    sourceMetrics.setCatalogProviders(Collections.singletonList(catalogProvider));
    sourceMetrics.setFederatedSources(Collections.singletonList(fedSource));

    assertThat(sourceMetrics, not(nullValue()));

    return sourceMetrics;
  }
  @Test
  public void testSourceCreatedWithNullId() throws Exception {
    String sourceId = null;
    String metricName = SourceMetrics.QUERIES_TOTAL_RESULTS_SCOPE;

    sourceMetrics = configureSourceMetrics(sourceId);

    // Simulate initial creation of Source
    addSource();

    // Simulate changing Source's name
    String newSourceId = "cp-1";
    when(catalogProvider.getId()).thenReturn(newSourceId);

    sourceMetrics.updateMetric(newSourceId, metricName, 1);

    assertMetricCount(newSourceId, metricName, 1);
  }
  @Test
  public void testDeleteSourceBlankSourceId() throws Exception {
    String sourceId = "cp-1";
    String metricName = SourceMetrics.QUERIES_SCOPE;

    sourceMetrics = configureSourceMetrics(sourceId);
    addSource();

    // Simulate Source returning empty sourceId
    when(catalogProvider.getId()).thenReturn("");

    sourceMetrics.deletingSource(catalogProvider, null);

    String key = sourceId + "." + metricName;
    SourceMetric sourceMetric = sourceMetrics.metrics.get(key);
    assertThat(sourceMetric, not(nullValue()));

    sourceMetrics.deletingSource(null, null);

    key = sourceId + "." + metricName;
    sourceMetric = sourceMetrics.metrics.get(key);
    assertThat(sourceMetric, not(nullValue()));
  }
Exemple #5
0
  @Override
  protected Object executeWithSubject() throws Exception {

    List<CatalogProvider> providers = getCatalogProviders();

    if (providers.isEmpty() || providers.size() < 2) {
      console.println("Not enough CatalogProviders installed to migrate");
      return null;
    }

    console.println("The \"FROM\" provider is: " + providers.get(0).getClass().getSimpleName());
    CatalogProvider provider = providers.get(1);
    console.println("The \"TO\" provider is: " + provider.getClass().getSimpleName());
    String answer = getInput("Do you wish to continue? (yes/no)");
    if (!"yes".equalsIgnoreCase(answer)) {
      console.println();
      console.println("Now exiting...");
      console.flush();
      return null;
    }

    ingestProvider = new Provider(provider);

    framework = getCatalog();

    start = System.currentTimeMillis();

    final Filter filter =
        (cqlFilter != null)
            ? CQL.toFilter(cqlFilter)
            : getFilter(getFilterStartTime(start), start, Metacard.MODIFIED);

    QueryImpl query = new QueryImpl(filter);
    query.setRequestsTotalResultsCount(true);
    query.setPageSize(batchSize);
    query.setSortBy(new SortByImpl(Metacard.MODIFIED, SortOrder.DESCENDING));
    QueryRequest queryRequest = new QueryRequestImpl(query);
    SourceResponse response;
    try {
      response = framework.query(queryRequest);
    } catch (FederationException e) {
      printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
      return null;
    } catch (SourceUnavailableException e) {
      printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
      return null;
    } catch (UnsupportedQueryException e) {
      printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
      return null;
    }
    final long totalHits = response.getHits();
    final long totalPossible;
    if (totalHits == 0) {
      console.println("No records were found to replicate.");
      return null;
    }

    // If the maxMetacards is set, restrict the totalPossible to the number of maxMetacards
    if (maxMetacards > 0 && maxMetacards <= totalHits) {
      totalPossible = maxMetacards;
    } else {
      totalPossible = totalHits;
    }

    console.println("Starting migration for " + totalPossible + " Records");

    if (multithreaded > 1 && totalPossible > batchSize) {
      BlockingQueue<Runnable> blockingQueue = new ArrayBlockingQueue<Runnable>(multithreaded);
      RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy();
      final ExecutorService executorService =
          new ThreadPoolExecutor(
              multithreaded,
              multithreaded,
              0L,
              TimeUnit.MILLISECONDS,
              blockingQueue,
              rejectedExecutionHandler);
      console.printf("Running %d threads during replication.%n", multithreaded);

      do {
        LOGGER.debug("In loop at iteration {}", queryIndex.get());
        executorService.submit(
            () -> {
              int count = queryAndIngest(framework, ingestProvider, queryIndex.get(), filter);
              printProgressAndFlush(start, totalPossible, ingestCount.addAndGet(count));
            });
      } while (queryIndex.addAndGet(batchSize) <= totalPossible);
      executorService.shutdown();

      while (!executorService.isTerminated()) {
        try {
          TimeUnit.SECONDS.sleep(1);
        } catch (InterruptedException e) {
          // ignore
        }
      }
    } else {
      do {
        int count = queryAndIngest(framework, ingestProvider, queryIndex.get(), filter);
        printProgressAndFlush(start, totalPossible, ingestCount.addAndGet(count));
      } while (queryIndex.addAndGet(batchSize) <= totalPossible);
    }

    console.println();
    long end = System.currentTimeMillis();
    String completed =
        String.format(
            " %d record(s) replicated; %d record(s) failed; completed in %3.3f seconds.",
            ingestCount.get(), failedCount.get(), (end - start) / MS_PER_SECOND);
    LOGGER.info("Replication Complete: {}", completed);
    console.println(completed);

    return null;
  }