@Override
protected List<Metacard> query(CatalogFacade framework, int startIndex, Filter filter) {
QueryImpl query = new QueryImpl(filter);
query.setRequestsTotalResultsCount(false);
query.setPageSize(batchSize);
query.setSortBy(new SortByImpl(Metacard.MODIFIED, SortOrder.DESCENDING));
QueryRequest queryRequest = new QueryRequestImpl(query);
query.setStartIndex(startIndex);
SourceResponse response;
try {
LOGGER.debug("Querying with startIndex: {}", startIndex);
response = framework.query(queryRequest);
} catch (UnsupportedQueryException e) {
printErrorMessage(String.format("Received error from Framework: %s%n", e.getMessage()));
return null;
} catch (SourceUnavailableException e) {
printErrorMessage(String.format("Received error from Frameworks: %s%n", e.getMessage()));
return null;
} catch (FederationException e) {
printErrorMessage(String.format("Received error from Frameworks: %s%n", e.getMessage()));
return null;
}
if (response.getProcessingDetails() != null && !response.getProcessingDetails().isEmpty()) {
for (SourceProcessingDetails details : response.getProcessingDetails()) {
LOGGER.debug("Got Issues: {}", details.getWarnings());
}
return null;
}
return response.getResults().stream().map(Result::getMetacard).collect(Collectors.toList());
}
@Override
protected Object executeWithSubject() throws Exception {
List<CatalogProvider> providers = getCatalogProviders();
if (providers.isEmpty() || providers.size() < 2) {
console.println("Not enough CatalogProviders installed to migrate");
return null;
}
console.println("The \"FROM\" provider is: " + providers.get(0).getClass().getSimpleName());
CatalogProvider provider = providers.get(1);
console.println("The \"TO\" provider is: " + provider.getClass().getSimpleName());
String answer = getInput("Do you wish to continue? (yes/no)");
if (!"yes".equalsIgnoreCase(answer)) {
console.println();
console.println("Now exiting...");
console.flush();
return null;
}
ingestProvider = new Provider(provider);
framework = getCatalog();
start = System.currentTimeMillis();
final Filter filter =
(cqlFilter != null)
? CQL.toFilter(cqlFilter)
: getFilter(getFilterStartTime(start), start, Metacard.MODIFIED);
QueryImpl query = new QueryImpl(filter);
query.setRequestsTotalResultsCount(true);
query.setPageSize(batchSize);
query.setSortBy(new SortByImpl(Metacard.MODIFIED, SortOrder.DESCENDING));
QueryRequest queryRequest = new QueryRequestImpl(query);
SourceResponse response;
try {
response = framework.query(queryRequest);
} catch (FederationException e) {
printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
return null;
} catch (SourceUnavailableException e) {
printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
return null;
} catch (UnsupportedQueryException e) {
printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
return null;
}
final long totalHits = response.getHits();
final long totalPossible;
if (totalHits == 0) {
console.println("No records were found to replicate.");
return null;
}
// If the maxMetacards is set, restrict the totalPossible to the number of maxMetacards
if (maxMetacards > 0 && maxMetacards <= totalHits) {
totalPossible = maxMetacards;
} else {
totalPossible = totalHits;
}
console.println("Starting migration for " + totalPossible + " Records");
if (multithreaded > 1 && totalPossible > batchSize) {
BlockingQueue<Runnable> blockingQueue = new ArrayBlockingQueue<Runnable>(multithreaded);
RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy();
final ExecutorService executorService =
new ThreadPoolExecutor(
multithreaded,
multithreaded,
0L,
TimeUnit.MILLISECONDS,
blockingQueue,
rejectedExecutionHandler);
console.printf("Running %d threads during replication.%n", multithreaded);
do {
LOGGER.debug("In loop at iteration {}", queryIndex.get());
executorService.submit(
() -> {
int count = queryAndIngest(framework, ingestProvider, queryIndex.get(), filter);
printProgressAndFlush(start, totalPossible, ingestCount.addAndGet(count));
});
} while (queryIndex.addAndGet(batchSize) <= totalPossible);
executorService.shutdown();
while (!executorService.isTerminated()) {
try {
TimeUnit.SECONDS.sleep(1);
} catch (InterruptedException e) {
// ignore
}
}
} else {
do {
int count = queryAndIngest(framework, ingestProvider, queryIndex.get(), filter);
printProgressAndFlush(start, totalPossible, ingestCount.addAndGet(count));
} while (queryIndex.addAndGet(batchSize) <= totalPossible);
}
console.println();
long end = System.currentTimeMillis();
String completed =
String.format(
" %d record(s) replicated; %d record(s) failed; completed in %3.3f seconds.",
ingestCount.get(), failedCount.get(), (end - start) / MS_PER_SECOND);
LOGGER.info("Replication Complete: {}", completed);
console.println(completed);
return null;
}
