@Override protected List<Metacard> query(CatalogFacade framework, int startIndex, Filter filter) { QueryImpl query = new QueryImpl(filter); query.setRequestsTotalResultsCount(false); query.setPageSize(batchSize); query.setSortBy(new SortByImpl(Metacard.MODIFIED, SortOrder.DESCENDING)); QueryRequest queryRequest = new QueryRequestImpl(query); query.setStartIndex(startIndex); SourceResponse response; try { LOGGER.debug("Querying with startIndex: {}", startIndex); response = framework.query(queryRequest); } catch (UnsupportedQueryException e) { printErrorMessage(String.format("Received error from Framework: %s%n", e.getMessage())); return null; } catch (SourceUnavailableException e) { printErrorMessage(String.format("Received error from Frameworks: %s%n", e.getMessage())); return null; } catch (FederationException e) { printErrorMessage(String.format("Received error from Frameworks: %s%n", e.getMessage())); return null; } if (response.getProcessingDetails() != null && !response.getProcessingDetails().isEmpty()) { for (SourceProcessingDetails details : response.getProcessingDetails()) { LOGGER.debug("Got Issues: {}", details.getWarnings()); } return null; } return response.getResults().stream().map(Result::getMetacard).collect(Collectors.toList()); }
@Override protected Object executeWithSubject() throws Exception { List<CatalogProvider> providers = getCatalogProviders(); if (providers.isEmpty() || providers.size() < 2) { console.println("Not enough CatalogProviders installed to migrate"); return null; } console.println("The \"FROM\" provider is: " + providers.get(0).getClass().getSimpleName()); CatalogProvider provider = providers.get(1); console.println("The \"TO\" provider is: " + provider.getClass().getSimpleName()); String answer = getInput("Do you wish to continue? (yes/no)"); if (!"yes".equalsIgnoreCase(answer)) { console.println(); console.println("Now exiting..."); console.flush(); return null; } ingestProvider = new Provider(provider); framework = getCatalog(); start = System.currentTimeMillis(); final Filter filter = (cqlFilter != null) ? CQL.toFilter(cqlFilter) : getFilter(getFilterStartTime(start), start, Metacard.MODIFIED); QueryImpl query = new QueryImpl(filter); query.setRequestsTotalResultsCount(true); query.setPageSize(batchSize); query.setSortBy(new SortByImpl(Metacard.MODIFIED, SortOrder.DESCENDING)); QueryRequest queryRequest = new QueryRequestImpl(query); SourceResponse response; try { response = framework.query(queryRequest); } catch (FederationException e) { printErrorMessage("Error occurred while querying the Framework." + e.getMessage()); return null; } catch (SourceUnavailableException e) { printErrorMessage("Error occurred while querying the Framework." + e.getMessage()); return null; } catch (UnsupportedQueryException e) { printErrorMessage("Error occurred while querying the Framework." + e.getMessage()); return null; } final long totalHits = response.getHits(); final long totalPossible; if (totalHits == 0) { console.println("No records were found to replicate."); return null; } // If the maxMetacards is set, restrict the totalPossible to the number of maxMetacards if (maxMetacards > 0 && maxMetacards <= totalHits) { totalPossible = maxMetacards; } else { totalPossible = totalHits; } console.println("Starting migration for " + totalPossible + " Records"); if (multithreaded > 1 && totalPossible > batchSize) { BlockingQueue<Runnable> blockingQueue = new ArrayBlockingQueue<Runnable>(multithreaded); RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy(); final ExecutorService executorService = new ThreadPoolExecutor( multithreaded, multithreaded, 0L, TimeUnit.MILLISECONDS, blockingQueue, rejectedExecutionHandler); console.printf("Running %d threads during replication.%n", multithreaded); do { LOGGER.debug("In loop at iteration {}", queryIndex.get()); executorService.submit( () -> { int count = queryAndIngest(framework, ingestProvider, queryIndex.get(), filter); printProgressAndFlush(start, totalPossible, ingestCount.addAndGet(count)); }); } while (queryIndex.addAndGet(batchSize) <= totalPossible); executorService.shutdown(); while (!executorService.isTerminated()) { try { TimeUnit.SECONDS.sleep(1); } catch (InterruptedException e) { // ignore } } } else { do { int count = queryAndIngest(framework, ingestProvider, queryIndex.get(), filter); printProgressAndFlush(start, totalPossible, ingestCount.addAndGet(count)); } while (queryIndex.addAndGet(batchSize) <= totalPossible); } console.println(); long end = System.currentTimeMillis(); String completed = String.format( " %d record(s) replicated; %d record(s) failed; completed in %3.3f seconds.", ingestCount.get(), failedCount.get(), (end - start) / MS_PER_SECOND); LOGGER.info("Replication Complete: {}", completed); console.println(completed); return null; }