public CatalogProvider waitForCatalogProvider() throws InterruptedException {
LOGGER.info("Waiting for CatalogProvider to become available.");
serviceManager.printInactiveBundles();
CatalogProvider provider = null;
long timeoutLimit = System.currentTimeMillis() + CATALOG_PROVIDER_TIMEOUT;
boolean available = false;
while (!available) {
if (provider == null) {
ServiceReference<CatalogProvider> providerRef =
serviceManager.getServiceReference(CatalogProvider.class);
if (providerRef != null) {
provider = serviceManager.getService(providerRef);
}
}
if (provider != null) {
available = provider.isAvailable();
}
if (!available) {
if (System.currentTimeMillis() > timeoutLimit) {
LOGGER.info("CatalogProvider.isAvailable = false");
serviceManager.printInactiveBundles();
fail(
String.format(
"Catalog provider timed out after %d minutes.",
TimeUnit.MILLISECONDS.toMinutes(CATALOG_PROVIDER_TIMEOUT)));
}
Thread.sleep(100);
}
}
LOGGER.info("CatalogProvider is available.");
return provider;
}
/** ********************************************************************************* */
private SourceMetricsImpl configureSourceMetrics(String sourceId) throws Exception {
catalogProvider = mock(CatalogProvider.class);
when(catalogProvider.getId()).thenReturn(sourceId);
fedSource = mock(FederatedSource.class);
when(fedSource.getId()).thenReturn("fs-1");
sourceMetrics = new SourceMetricsImpl();
sourceMetrics.setCatalogProviders(Collections.singletonList(catalogProvider));
sourceMetrics.setFederatedSources(Collections.singletonList(fedSource));
assertThat(sourceMetrics, not(nullValue()));
return sourceMetrics;
}
@Test
public void testSourceCreatedWithNullId() throws Exception {
String sourceId = null;
String metricName = SourceMetrics.QUERIES_TOTAL_RESULTS_SCOPE;
sourceMetrics = configureSourceMetrics(sourceId);
// Simulate initial creation of Source
addSource();
// Simulate changing Source's name
String newSourceId = "cp-1";
when(catalogProvider.getId()).thenReturn(newSourceId);
sourceMetrics.updateMetric(newSourceId, metricName, 1);
assertMetricCount(newSourceId, metricName, 1);
}
@Test
public void testDeleteSourceBlankSourceId() throws Exception {
String sourceId = "cp-1";
String metricName = SourceMetrics.QUERIES_SCOPE;
sourceMetrics = configureSourceMetrics(sourceId);
addSource();
// Simulate Source returning empty sourceId
when(catalogProvider.getId()).thenReturn("");
sourceMetrics.deletingSource(catalogProvider, null);
String key = sourceId + "." + metricName;
SourceMetric sourceMetric = sourceMetrics.metrics.get(key);
assertThat(sourceMetric, not(nullValue()));
sourceMetrics.deletingSource(null, null);
key = sourceId + "." + metricName;
sourceMetric = sourceMetrics.metrics.get(key);
assertThat(sourceMetric, not(nullValue()));
}
@Override
protected Object executeWithSubject() throws Exception {
List<CatalogProvider> providers = getCatalogProviders();
if (providers.isEmpty() || providers.size() < 2) {
console.println("Not enough CatalogProviders installed to migrate");
return null;
}
console.println("The \"FROM\" provider is: " + providers.get(0).getClass().getSimpleName());
CatalogProvider provider = providers.get(1);
console.println("The \"TO\" provider is: " + provider.getClass().getSimpleName());
String answer = getInput("Do you wish to continue? (yes/no)");
if (!"yes".equalsIgnoreCase(answer)) {
console.println();
console.println("Now exiting...");
console.flush();
return null;
}
ingestProvider = new Provider(provider);
framework = getCatalog();
start = System.currentTimeMillis();
final Filter filter =
(cqlFilter != null)
? CQL.toFilter(cqlFilter)
: getFilter(getFilterStartTime(start), start, Metacard.MODIFIED);
QueryImpl query = new QueryImpl(filter);
query.setRequestsTotalResultsCount(true);
query.setPageSize(batchSize);
query.setSortBy(new SortByImpl(Metacard.MODIFIED, SortOrder.DESCENDING));
QueryRequest queryRequest = new QueryRequestImpl(query);
SourceResponse response;
try {
response = framework.query(queryRequest);
} catch (FederationException e) {
printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
return null;
} catch (SourceUnavailableException e) {
printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
return null;
} catch (UnsupportedQueryException e) {
printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
return null;
}
final long totalHits = response.getHits();
final long totalPossible;
if (totalHits == 0) {
console.println("No records were found to replicate.");
return null;
}
// If the maxMetacards is set, restrict the totalPossible to the number of maxMetacards
if (maxMetacards > 0 && maxMetacards <= totalHits) {
totalPossible = maxMetacards;
} else {
totalPossible = totalHits;
}
console.println("Starting migration for " + totalPossible + " Records");
if (multithreaded > 1 && totalPossible > batchSize) {
BlockingQueue<Runnable> blockingQueue = new ArrayBlockingQueue<Runnable>(multithreaded);
RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy();
final ExecutorService executorService =
new ThreadPoolExecutor(
multithreaded,
multithreaded,
0L,
TimeUnit.MILLISECONDS,
blockingQueue,
rejectedExecutionHandler);
console.printf("Running %d threads during replication.%n", multithreaded);
do {
LOGGER.debug("In loop at iteration {}", queryIndex.get());
executorService.submit(
() -> {
int count = queryAndIngest(framework, ingestProvider, queryIndex.get(), filter);
printProgressAndFlush(start, totalPossible, ingestCount.addAndGet(count));
});
} while (queryIndex.addAndGet(batchSize) <= totalPossible);
executorService.shutdown();
while (!executorService.isTerminated()) {
try {
TimeUnit.SECONDS.sleep(1);
} catch (InterruptedException e) {
// ignore
}
}
} else {
do {
int count = queryAndIngest(framework, ingestProvider, queryIndex.get(), filter);
printProgressAndFlush(start, totalPossible, ingestCount.addAndGet(count));
} while (queryIndex.addAndGet(batchSize) <= totalPossible);
}
console.println();
long end = System.currentTimeMillis();
String completed =
String.format(
" %d record(s) replicated; %d record(s) failed; completed in %3.3f seconds.",
ingestCount.get(), failedCount.get(), (end - start) / MS_PER_SECOND);
LOGGER.info("Replication Complete: {}", completed);
console.println(completed);
return null;
}