@Override
protected List<Metacard> query(CatalogFacade framework, int startIndex, Filter filter) {
QueryImpl query = new QueryImpl(filter);
query.setRequestsTotalResultsCount(false);
query.setPageSize(batchSize);
query.setSortBy(new SortByImpl(Metacard.MODIFIED, SortOrder.DESCENDING));
QueryRequest queryRequest = new QueryRequestImpl(query);
query.setStartIndex(startIndex);
SourceResponse response;
try {
LOGGER.debug("Querying with startIndex: {}", startIndex);
response = framework.query(queryRequest);
} catch (UnsupportedQueryException e) {
printErrorMessage(String.format("Received error from Framework: %s%n", e.getMessage()));
return null;
} catch (SourceUnavailableException e) {
printErrorMessage(String.format("Received error from Frameworks: %s%n", e.getMessage()));
return null;
} catch (FederationException e) {
printErrorMessage(String.format("Received error from Frameworks: %s%n", e.getMessage()));
return null;
}
if (response.getProcessingDetails() != null && !response.getProcessingDetails().isEmpty()) {
for (SourceProcessingDetails details : response.getProcessingDetails()) {
LOGGER.debug("Got Issues: {}", details.getWarnings());
}
return null;
}
return response.getResults().stream().map(Result::getMetacard).collect(Collectors.toList());
}
@Test
public void testResultNumReturnedIsZero()
throws WfsException, TransformerConfigurationException, UnsupportedQueryException {
// Setup
final String TITLE = "title";
final String searchPhrase = "*";
final int pageSize = 1;
WfsSource source =
getWfsSource(
ONE_TEXT_PROPERTY_SCHEMA,
MockWfsServer.getFilterCapabilities(),
Wfs20Constants.EPSG_4326_URN,
3,
false,
true,
0);
QueryImpl query =
new QueryImpl(builder.attribute(Metacard.ANY_TEXT).is().like().text(searchPhrase));
query.setPageSize(pageSize);
SortBy sortBy = new SortByImpl(TITLE, SortOrder.DESCENDING);
query.setSortBy(sortBy);
QueryRequestImpl queryReq = new QueryRequestImpl(query);
// Perform test
SourceResponse resp = source.query(queryReq);
assertEquals(3, resp.getResults().size());
}
private CswRecordCollection buildCollection(
SourceResponse sourceResponse, Map<String, Serializable> arguments) {
CswRecordCollection recordCollection = new CswRecordCollection();
recordCollection.setNumberOfRecordsMatched(sourceResponse.getHits());
recordCollection.setNumberOfRecordsReturned(sourceResponse.getResults().size());
recordCollection.setStartPosition(sourceResponse.getRequest().getQuery().getStartIndex());
Object elementSetTypeArg = arguments.get(CswConstants.ELEMENT_SET_TYPE);
if (elementSetTypeArg instanceof ElementSetType) {
ElementSetType elementSetType = (ElementSetType) elementSetTypeArg;
recordCollection.setElementSetType(elementSetType);
}
Object elementNamesArg = arguments.get(CswConstants.ELEMENT_NAMES);
if (elementNamesArg instanceof QName[]) {
QName[] qnames = (QName[]) elementNamesArg;
if (qnames.length > 0) {
List<QName> elementNames = new ArrayList();
for (QName entry : qnames) {
elementNames.add(entry);
}
recordCollection.setElementName(elementNames);
}
}
Object isByIdQuery = arguments.get(CswConstants.IS_BY_ID_QUERY);
if (isByIdQuery != null) {
recordCollection.setById((Boolean) isByIdQuery);
}
Object arg = arguments.get((CswConstants.GET_RECORDS));
if (arg != null && arg instanceof GetRecordsType) {
recordCollection.setRequest((GetRecordsType) arg);
}
Object resultType = arguments.get(CswConstants.RESULT_TYPE_PARAMETER);
if (resultType instanceof ResultType) {
recordCollection.setResultType((ResultType) resultType);
}
Object outputSchema = arguments.get(CswConstants.OUTPUT_SCHEMA_PARAMETER);
if (outputSchema instanceof String) {
recordCollection.setOutputSchema((String) outputSchema);
} else {
recordCollection.setOutputSchema(CswConstants.CSW_OUTPUT_SCHEMA);
}
Object doWriteNamespaces = arguments.get(CswConstants.WRITE_NAMESPACES);
if (doWriteNamespaces instanceof Boolean) {
recordCollection.setDoWriteNamespaces((Boolean) doWriteNamespaces);
}
return recordCollection;
}
private void validateInput(SourceResponse sourceResponse, Map<String, Serializable> arguments) {
if (null == sourceResponse) {
throw new IllegalArgumentException("Null source response.");
} else if (null == arguments) {
throw new IllegalArgumentException("Null argument map.");
} else if (null == sourceResponse.getResults()) {
throw new IllegalArgumentException("Null results list.");
} else if (!isByIdQuery(arguments)
&& null == arguments.get(CswConstants.RESULT_TYPE_PARAMETER)) {
// An exception is thrown only if the query isn't by ID (i.e. it's not a GetRecordById
// request) because GetRecordById does not use the ResultType attribute.
throw new IllegalArgumentException("Null result type argument.");
} else if (null == sourceResponse.getRequest()) {
throw new IllegalArgumentException("Null source response query request.");
} else if (null == sourceResponse.getRequest().getQuery()) {
throw new IllegalArgumentException("Null source response query.");
}
}
@Override
public BinaryContent transform(SourceResponse sourceResponse, Map<String, Serializable> arguments)
throws CatalogTransformerException {
validateInput(sourceResponse, arguments);
CswRecordCollection recordCollection = buildCollection(sourceResponse, arguments);
ByteArrayInputStream bais;
if (ResultType.VALIDATE.equals(recordCollection.getResultType())) {
ByteArrayOutputStream baos = writeAcknowledgement(recordCollection.getRequest());
bais = new ByteArrayInputStream(baos.toByteArray());
} else {
// "catches" recordCollection.getResultType() == null
List<Result> results = sourceResponse.getResults();
String xmlString = convert(recordCollection, results, arguments);
bais = new ByteArrayInputStream(xmlString.getBytes(StandardCharsets.UTF_8));
}
BinaryContent transformedContent = new BinaryContentImpl(bais, new MimeType());
return transformedContent;
}
@Override
protected Object doExecute() throws Exception {
String formatString =
"%1$-33s %2$-26s %3$-" + TITLE_MAX_LENGTH + "s %4$-" + EXCERPT_MAX_LENGTH + "s%n";
CatalogFacade catalogProvider = getCatalog();
Filter filter = null;
if (cqlFilter != null) {
filter = CQL.toFilter(cqlFilter);
} else {
if (searchPhrase == null) {
searchPhrase = "*";
}
if (caseSensitive) {
filter =
getFilterBuilder()
.attribute(Metacard.ANY_TEXT)
.is()
.like()
.caseSensitiveText(searchPhrase);
} else {
filter = getFilterBuilder().attribute(Metacard.ANY_TEXT).is().like().text(searchPhrase);
}
}
QueryImpl query = new QueryImpl(filter);
query.setRequestsTotalResultsCount(true);
if (numberOfItems > -1) {
query.setPageSize(numberOfItems);
}
long start = System.currentTimeMillis();
SourceResponse response = catalogProvider.query(new QueryRequestImpl(query));
long end = System.currentTimeMillis();
int size = 0;
if (response.getResults() != null) {
size = response.getResults().size();
}
console.println();
console.printf(
" %d result(s) out of %s%d%s in %3.3f seconds",
(size),
Ansi.ansi().fg(Ansi.Color.CYAN).toString(),
response.getHits(),
Ansi.ansi().reset().toString(),
(end - start) / MILLISECONDS_PER_SECOND);
console.printf(formatString, "", "", "", "");
printHeaderMessage(String.format(formatString, ID, DATE, TITLE, EXCERPT));
for (Result result : response.getResults()) {
Metacard metacard = result.getMetacard();
String title = (metacard.getTitle() != null ? metacard.getTitle() : "N/A");
String excerpt = "N/A";
String modifiedDate = "";
if (searchPhrase != null) {
if (metacard.getMetadata() != null) {
XPathHelper helper = new XPathHelper(metacard.getMetadata());
String indexedText = helper.getDocument().getDocumentElement().getTextContent();
indexedText = indexedText.replaceAll("\\r\\n|\\r|\\n", " ");
String normalizedSearchPhrase = searchPhrase.replaceAll("\\*", "");
int index = -1;
if (caseSensitive) {
index = indexedText.indexOf(normalizedSearchPhrase);
} else {
index = indexedText.toLowerCase().indexOf(normalizedSearchPhrase.toLowerCase());
}
if (index != -1) {
int contextLength = (EXCERPT_MAX_LENGTH - normalizedSearchPhrase.length() - 8) / 2;
excerpt = "..." + indexedText.substring(Math.max(index - contextLength, 0), index);
excerpt = excerpt + Ansi.ansi().fg(Ansi.Color.GREEN).toString();
excerpt =
excerpt + indexedText.substring(index, index + normalizedSearchPhrase.length());
excerpt = excerpt + Ansi.ansi().reset().toString();
excerpt =
excerpt
+ indexedText.substring(
index + normalizedSearchPhrase.length(),
Math.min(
indexedText.length(),
index + normalizedSearchPhrase.length() + contextLength))
+ "...";
}
}
}
if (metacard.getModifiedDate() != null) {
modifiedDate =
new DateTime(metacard.getModifiedDate().getTime()).toString(DATETIME_FORMATTER);
}
console.printf(
formatString,
metacard.getId(),
modifiedDate,
title.substring(0, Math.min(title.length(), TITLE_MAX_LENGTH)),
excerpt);
}
return null;
}
@Override
protected Object executeWithSubject() throws Exception {
List<CatalogProvider> providers = getCatalogProviders();
if (providers.isEmpty() || providers.size() < 2) {
console.println("Not enough CatalogProviders installed to migrate");
return null;
}
console.println("The \"FROM\" provider is: " + providers.get(0).getClass().getSimpleName());
CatalogProvider provider = providers.get(1);
console.println("The \"TO\" provider is: " + provider.getClass().getSimpleName());
String answer = getInput("Do you wish to continue? (yes/no)");
if (!"yes".equalsIgnoreCase(answer)) {
console.println();
console.println("Now exiting...");
console.flush();
return null;
}
ingestProvider = new Provider(provider);
framework = getCatalog();
start = System.currentTimeMillis();
final Filter filter =
(cqlFilter != null)
? CQL.toFilter(cqlFilter)
: getFilter(getFilterStartTime(start), start, Metacard.MODIFIED);
QueryImpl query = new QueryImpl(filter);
query.setRequestsTotalResultsCount(true);
query.setPageSize(batchSize);
query.setSortBy(new SortByImpl(Metacard.MODIFIED, SortOrder.DESCENDING));
QueryRequest queryRequest = new QueryRequestImpl(query);
SourceResponse response;
try {
response = framework.query(queryRequest);
} catch (FederationException e) {
printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
return null;
} catch (SourceUnavailableException e) {
printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
return null;
} catch (UnsupportedQueryException e) {
printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
return null;
}
final long totalHits = response.getHits();
final long totalPossible;
if (totalHits == 0) {
console.println("No records were found to replicate.");
return null;
}
// If the maxMetacards is set, restrict the totalPossible to the number of maxMetacards
if (maxMetacards > 0 && maxMetacards <= totalHits) {
totalPossible = maxMetacards;
} else {
totalPossible = totalHits;
}
console.println("Starting migration for " + totalPossible + " Records");
if (multithreaded > 1 && totalPossible > batchSize) {
BlockingQueue<Runnable> blockingQueue = new ArrayBlockingQueue<Runnable>(multithreaded);
RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy();
final ExecutorService executorService =
new ThreadPoolExecutor(
multithreaded,
multithreaded,
0L,
TimeUnit.MILLISECONDS,
blockingQueue,
rejectedExecutionHandler);
console.printf("Running %d threads during replication.%n", multithreaded);
do {
LOGGER.debug("In loop at iteration {}", queryIndex.get());
executorService.submit(
() -> {
int count = queryAndIngest(framework, ingestProvider, queryIndex.get(), filter);
printProgressAndFlush(start, totalPossible, ingestCount.addAndGet(count));
});
} while (queryIndex.addAndGet(batchSize) <= totalPossible);
executorService.shutdown();
while (!executorService.isTerminated()) {
try {
TimeUnit.SECONDS.sleep(1);
} catch (InterruptedException e) {
// ignore
}
}
} else {
do {
int count = queryAndIngest(framework, ingestProvider, queryIndex.get(), filter);
printProgressAndFlush(start, totalPossible, ingestCount.addAndGet(count));
} while (queryIndex.addAndGet(batchSize) <= totalPossible);
}
console.println();
long end = System.currentTimeMillis();
String completed =
String.format(
" %d record(s) replicated; %d record(s) failed; completed in %3.3f seconds.",
ingestCount.get(), failedCount.get(), (end - start) / MS_PER_SECOND);
LOGGER.info("Replication Complete: {}", completed);
console.println(completed);
return null;
}
@SuppressWarnings({"rawtypes", "unchecked"})
@Override
public void run() {
String methodName = "run";
logger.entry(methodName);
SortBy sortBy = query.getSortBy();
// Prepare the Comparators that we will use
Comparator<Result> coreComparator = DEFAULT_COMPARATOR;
if (sortBy != null && sortBy.getPropertyName() != null) {
PropertyName sortingProp = sortBy.getPropertyName();
String sortType = sortingProp.getPropertyName();
SortOrder sortOrder =
(sortBy.getSortOrder() == null) ? SortOrder.DESCENDING : sortBy.getSortOrder();
logger.debug("Sorting by type: " + sortType);
logger.debug("Sorting by Order: " + sortBy.getSortOrder());
// Temporal searches are currently sorted by the effective time
if (Metacard.EFFECTIVE.equals(sortType) || Result.TEMPORAL.equals(sortType)) {
coreComparator = new TemporalResultComparator(sortOrder);
} else if (Result.DISTANCE.equals(sortType)) {
coreComparator = new DistanceResultComparator(sortOrder);
} else if (Result.RELEVANCE.equals(sortType)) {
coreComparator = new RelevanceResultComparator(sortOrder);
}
}
List<Result> resultList = new ArrayList<Result>();
long totalHits = 0;
Set<ProcessingDetails> processingDetails = returnResults.getProcessingDetails();
Map<String, Serializable> returnProperties = returnResults.getProperties();
for (final Entry<Source, Future<SourceResponse>> entry : futures.entrySet()) {
Source site = entry.getKey();
SourceResponse sourceResponse = null;
try {
sourceResponse =
query.getTimeoutMillis() < 1
? entry.getValue().get()
: entry.getValue().get(getTimeRemaining(deadline), TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
logger.warn(
"Couldn't get results from completed federated query on site with ShortName "
+ site.getId(),
e);
processingDetails.add(new ProcessingDetailsImpl(site.getId(), e));
} catch (ExecutionException e) {
logger.warn(
"Couldn't get results from completed federated query on site " + site.getId(), e);
if (logger.isDebugEnabled()) {
logger.debug("Adding exception to response.");
}
processingDetails.add(new ProcessingDetailsImpl(site.getId(), e));
} catch (TimeoutException e) {
logger.warn("search timed out: " + new Date() + " on site " + site.getId());
processingDetails.add(new ProcessingDetailsImpl(site.getId(), e));
}
if (sourceResponse != null) {
List<Result> sourceResults = sourceResponse.getResults();
resultList.addAll(sourceResults);
long sourceHits = sourceResponse.getHits();
totalHits += sourceHits;
Map<String, Serializable> newSourceProperties = new HashMap<String, Serializable>();
newSourceProperties.put(QueryResponse.TOTAL_HITS, sourceHits);
newSourceProperties.put(QueryResponse.TOTAL_RESULTS_RETURNED, sourceResults.size());
Map<String, Serializable> originalSourceProperties = sourceResponse.getProperties();
if (originalSourceProperties != null) {
Serializable object = originalSourceProperties.get(QueryResponse.ELAPSED_TIME);
if (object != null && object instanceof Long) {
newSourceProperties.put(QueryResponse.ELAPSED_TIME, (Long) object);
originalSourceProperties.remove(QueryResponse.ELAPSED_TIME);
logger.debug(
"Setting the ellapsedTime responseProperty to {} for source {}",
object,
site.getId());
}
// TODO: for now add all properties into outgoing response's properties.
// this is not the best idea because we could get properties from records
// that get eliminated by the max results enforcement done below.
// See DDF-1183 for a possible solution.
returnProperties.putAll(originalSourceProperties);
}
returnProperties.put(site.getId(), (Serializable) newSourceProperties);
logger.debug("Setting the query responseProperties for site {}", site.getId());
// Add a List of siteIds so endpoints know what sites got queried
Serializable siteListObject = returnProperties.get(QueryResponse.SITE_LIST);
if (siteListObject != null && siteListObject instanceof List<?>) {
((List) siteListObject).add(site.getId());
} else {
siteListObject = new ArrayList<String>();
((List) siteListObject).add(site.getId());
returnProperties.put(QueryResponse.SITE_LIST, (Serializable) siteListObject);
}
}
}
logger.debug("all sites finished returning results: " + resultList.size());
Collections.sort(resultList, coreComparator);
returnResults.setHits(totalHits);
int maxResults = query.getPageSize() > 0 ? query.getPageSize() : Integer.MAX_VALUE;
returnResults.addResults(
resultList.size() > maxResults ? resultList.subList(0, maxResults) : resultList, true);
}
