@Override
protected List<Metacard> query(CatalogFacade framework, int startIndex, Filter filter) {
QueryImpl query = new QueryImpl(filter);
query.setRequestsTotalResultsCount(false);
query.setPageSize(batchSize);
query.setSortBy(new SortByImpl(Metacard.MODIFIED, SortOrder.DESCENDING));
QueryRequest queryRequest = new QueryRequestImpl(query);
query.setStartIndex(startIndex);
SourceResponse response;
try {
LOGGER.debug("Querying with startIndex: {}", startIndex);
response = framework.query(queryRequest);
} catch (UnsupportedQueryException e) {
printErrorMessage(String.format("Received error from Framework: %s%n", e.getMessage()));
return null;
} catch (SourceUnavailableException e) {
printErrorMessage(String.format("Received error from Frameworks: %s%n", e.getMessage()));
return null;
} catch (FederationException e) {
printErrorMessage(String.format("Received error from Frameworks: %s%n", e.getMessage()));
return null;
}
if (response.getProcessingDetails() != null && !response.getProcessingDetails().isEmpty()) {
for (SourceProcessingDetails details : response.getProcessingDetails()) {
LOGGER.debug("Got Issues: {}", details.getWarnings());
}
return null;
}
return response.getResults().stream().map(Result::getMetacard).collect(Collectors.toList());
}
@Test
public void testSortingNoSortBySortingSupported() throws Exception {
// Setup
final String searchPhrase = "*";
final int pageSize = 1;
WfsSource source =
getWfsSource(
ONE_TEXT_PROPERTY_SCHEMA,
MockWfsServer.getFilterCapabilities(),
Wfs20Constants.EPSG_4326_URN,
1,
false,
false,
0);
QueryImpl query =
new QueryImpl(builder.attribute(Metacard.ANY_TEXT).is().like().text(searchPhrase));
query.setPageSize(pageSize);
// Perform Test
GetFeatureType featureType = source.buildGetFeatureRequest(query);
// Verify
QueryType queryType = (QueryType) featureType.getAbstractQueryExpression().get(0).getValue();
assertFalse(queryType.isSetAbstractSortingClause());
}
@Test
public void testResultNumReturnedIsZero()
throws WfsException, TransformerConfigurationException, UnsupportedQueryException {
// Setup
final String TITLE = "title";
final String searchPhrase = "*";
final int pageSize = 1;
WfsSource source =
getWfsSource(
ONE_TEXT_PROPERTY_SCHEMA,
MockWfsServer.getFilterCapabilities(),
Wfs20Constants.EPSG_4326_URN,
3,
false,
true,
0);
QueryImpl query =
new QueryImpl(builder.attribute(Metacard.ANY_TEXT).is().like().text(searchPhrase));
query.setPageSize(pageSize);
SortBy sortBy = new SortByImpl(TITLE, SortOrder.DESCENDING);
query.setSortBy(sortBy);
QueryRequestImpl queryReq = new QueryRequestImpl(query);
// Perform test
SourceResponse resp = source.query(queryReq);
assertEquals(3, resp.getResults().size());
}
@Test
public void testSearchByMultipleTypes() throws Exception {
// Setup
int startIndex = 0;
int pageSize = 10;
WfsSource source =
getWfsSource(
ONE_TEXT_PROPERTY_SCHEMA,
MockWfsServer.getFilterCapabilities(),
Wfs20Constants.EPSG_4326_URN,
10,
false);
Filter filter0 =
builder.attribute(Metacard.CONTENT_TYPE).is().equalTo().text(SAMPLE_FEATURE_NAME + "8");
Filter filter1 =
builder.attribute(Metacard.CONTENT_TYPE).is().equalTo().text(SAMPLE_FEATURE_NAME + "9");
Filter filter2 = builder.attribute(Metacard.ANY_TEXT).is().like().text("*");
Filter typeSearchFilters = builder.anyOf(filter0, filter1);
QueryImpl query = new QueryImpl(builder.allOf(filter2, typeSearchFilters));
query.setPageSize(pageSize);
// Execute
GetFeatureType featureType = source.buildGetFeatureRequest(query);
int numTypes = featureType.getAbstractQueryExpression().size();
// Validate
assertEquals(2, numTypes);
}
/**
* Verify that the SortBy is NOT set. In this case, sorting is not supported in the capabilities.
*/
@Test
public void testSortingAscendingSortingNotSupported() throws Exception {
// Setup
final String searchPhrase = "*";
final String mockTemporalFeatureProperty = "myTemporalFeatureProperty";
final String mockFeatureType = "{http://example.com}" + SAMPLE_FEATURE_NAME + 0;
final int pageSize = 1;
// Set ImplementsSorting to FALSE (sorting not supported)
FilterCapabilities mockCapabilitiesSortingNotSupported = MockWfsServer.getFilterCapabilities();
ConformanceType conformance = mockCapabilitiesSortingNotSupported.getConformance();
List<DomainType> domainTypes = conformance.getConstraint();
for (DomainType domainType : domainTypes) {
if (StringUtils.equals(domainType.getName(), "ImplementsSorting")) {
ValueType valueType = new ValueType();
valueType.setValue("FALSE");
domainType.setDefaultValue(valueType);
break;
}
}
WfsSource source =
getWfsSource(
ONE_TEXT_PROPERTY_SCHEMA,
mockCapabilitiesSortingNotSupported,
Wfs20Constants.EPSG_4326_URN,
1,
false,
false,
0);
MetacardMapper mockMetacardMapper = mock(MetacardMapper.class);
when(mockMetacardMapper.getFeatureType()).thenReturn(mockFeatureType);
when(mockMetacardMapper.getSortByTemporalFeatureProperty())
.thenReturn(mockTemporalFeatureProperty);
List<MetacardMapper> mappers = new ArrayList<MetacardMapper>(1);
mappers.add(mockMetacardMapper);
source.setMetacardToFeatureMapper(mappers);
QueryImpl query =
new QueryImpl(builder.attribute(Metacard.ANY_TEXT).is().like().text(searchPhrase));
query.setPageSize(pageSize);
SortBy sortBy = new SortByImpl(Result.TEMPORAL, SortOrder.ASCENDING);
query.setSortBy(sortBy);
// Perform Test
GetFeatureType featureType = source.buildGetFeatureRequest(query);
// Verify
QueryType queryType = (QueryType) featureType.getAbstractQueryExpression().get(0).getValue();
assertFalse(queryType.isSetAbstractSortingClause());
}
/**
* Verify that the SortBy is set with the mapped Feature Property and a DESC sort order. In this
* case, the incoming sort property of TEMPORAL is mapped to myTemporalFeatureProperty.
*
* <p><?xml version="1.0" encoding="UTF-8" standalone="yes"?> <ns5:GetFeature startIndex="1"
* count="1" service="WFS" version="2.0.0" xmlns:ns2="http://www.opengis.net/ows/1.1"
* xmlns="http://www.opengis.net/fes/2.0" xmlns:ns4="http://www.opengis.net/gml"
* xmlns:ns3="http://www.w3.org/1999/xlink" xmlns:ns5="http://www.opengis.net/wfs/2.0"> <ns5:Query
* typeNames="SampleFeature0" handle="SampleFeature0"> <Filter> <PropertyIsLike wildCard="*"
* singleChar="?" escapeChar="!"> <Literal>*</Literal> <ValueReference>title</ValueReference>
* </PropertyIsLike> </Filter> <SortBy> <SortProperty>
* <ValueReference>myTemporalFeatureProperty</ValueReference> <SortOrder>DESC</SortOrder>
* </SortProperty> </SortBy> </ns5:Query> </ns5:GetFeature>
*/
@Test
public void testSortingDescendingSortingSupported() throws Exception {
// Setup
final String searchPhrase = "*";
final String mockTemporalFeatureProperty = "myTemporalFeatureProperty";
final String mockFeatureType = "{http://example.com}" + SAMPLE_FEATURE_NAME + 0;
final int pageSize = 1;
WfsSource source =
getWfsSource(
ONE_TEXT_PROPERTY_SCHEMA,
MockWfsServer.getFilterCapabilities(),
Wfs20Constants.EPSG_4326_URN,
1,
false,
false,
0);
MetacardMapper mockMetacardMapper = mock(MetacardMapper.class);
when(mockMetacardMapper.getFeatureType()).thenReturn(mockFeatureType);
when(mockMetacardMapper.getSortByTemporalFeatureProperty())
.thenReturn(mockTemporalFeatureProperty);
List<MetacardMapper> mappers = new ArrayList<MetacardMapper>(1);
mappers.add(mockMetacardMapper);
source.setMetacardToFeatureMapper(mappers);
QueryImpl query =
new QueryImpl(builder.attribute(Metacard.ANY_TEXT).is().like().text(searchPhrase));
query.setPageSize(pageSize);
SortBy sortBy = new SortByImpl(Result.TEMPORAL, SortOrder.DESCENDING);
query.setSortBy(sortBy);
// Perform Test
GetFeatureType featureType = source.buildGetFeatureRequest(query);
// Verify
QueryType queryType = (QueryType) featureType.getAbstractQueryExpression().get(0).getValue();
JAXBElement<?> abstractSortingClause = queryType.getAbstractSortingClause();
SortByType sortByType = (SortByType) abstractSortingClause.getValue();
assertThat(
sortByType.getSortProperty().get(0).getValueReference(), is(mockTemporalFeatureProperty));
assertThat(
sortByType.getSortProperty().get(0).getSortOrder().name(), is(SortOrderType.DESC.value()));
}
@Test
public void testTypeNameHasNoPrefix() throws WfsException, UnsupportedQueryException {
// Setup
final String TITLE = "title";
final String searchPhrase = "*";
final int pageSize = 1;
WfsSource source =
getWfsSource(
ONE_TEXT_PROPERTY_SCHEMA,
MockWfsServer.getFilterCapabilities(),
Wfs20Constants.EPSG_4326_URN,
3,
false,
false,
3);
QueryImpl query =
new QueryImpl(builder.attribute(Metacard.ANY_TEXT).is().like().text(searchPhrase));
query.setPageSize(pageSize);
SortBy sortBy = new SortByImpl(TITLE, SortOrder.DESCENDING);
query.setSortBy(sortBy);
// Perform test
GetFeatureType featureType = source.buildGetFeatureRequest(query);
// Validate
List<JAXBElement<?>> queryList = featureType.getAbstractQueryExpression();
for (JAXBElement<?> queryType : queryList) {
Object val = queryType.getValue();
QueryType queryTypeVal = (QueryType) val;
assertThat(queryTypeVal.getTypeNames().get(0), containsString("SampleFeature"));
assertThat(queryTypeVal.getTypeNames().get(0), is(not(containsString("Prefix"))));
assertThat(queryTypeVal.getTypeNames().get(0), is(not(containsString(":"))));
}
}
@Test
public void testSearchByType() throws Exception {
// Setup
int startIndex = 0;
int pageSize = 10;
WfsSource source =
getWfsSource(
ONE_TEXT_PROPERTY_SCHEMA,
MockWfsServer.getFilterCapabilities(),
Wfs20Constants.EPSG_4326_URN,
10,
false);
Filter filter =
builder.attribute(Metacard.CONTENT_TYPE).is().equalTo().text(SAMPLE_FEATURE_NAME + "0");
QueryImpl query = new QueryImpl(filter);
query.setPageSize(pageSize);
// Execute
GetFeatureType featureType = source.buildGetFeatureRequest(query);
QueryType queryType = (QueryType) featureType.getAbstractQueryExpression().get(0).getValue();
// Validate
assertEquals(SAMPLE_FEATURE_NAME + "0", queryType.getTypeNames().get(0));
}
@Override
protected Object doExecute() throws Exception {
String formatString =
"%1$-33s %2$-26s %3$-" + TITLE_MAX_LENGTH + "s %4$-" + EXCERPT_MAX_LENGTH + "s%n";
CatalogFacade catalogProvider = getCatalog();
Filter filter = null;
if (cqlFilter != null) {
filter = CQL.toFilter(cqlFilter);
} else {
if (searchPhrase == null) {
searchPhrase = "*";
}
if (caseSensitive) {
filter =
getFilterBuilder()
.attribute(Metacard.ANY_TEXT)
.is()
.like()
.caseSensitiveText(searchPhrase);
} else {
filter = getFilterBuilder().attribute(Metacard.ANY_TEXT).is().like().text(searchPhrase);
}
}
QueryImpl query = new QueryImpl(filter);
query.setRequestsTotalResultsCount(true);
if (numberOfItems > -1) {
query.setPageSize(numberOfItems);
}
long start = System.currentTimeMillis();
SourceResponse response = catalogProvider.query(new QueryRequestImpl(query));
long end = System.currentTimeMillis();
int size = 0;
if (response.getResults() != null) {
size = response.getResults().size();
}
console.println();
console.printf(
" %d result(s) out of %s%d%s in %3.3f seconds",
(size),
Ansi.ansi().fg(Ansi.Color.CYAN).toString(),
response.getHits(),
Ansi.ansi().reset().toString(),
(end - start) / MILLISECONDS_PER_SECOND);
console.printf(formatString, "", "", "", "");
printHeaderMessage(String.format(formatString, ID, DATE, TITLE, EXCERPT));
for (Result result : response.getResults()) {
Metacard metacard = result.getMetacard();
String title = (metacard.getTitle() != null ? metacard.getTitle() : "N/A");
String excerpt = "N/A";
String modifiedDate = "";
if (searchPhrase != null) {
if (metacard.getMetadata() != null) {
XPathHelper helper = new XPathHelper(metacard.getMetadata());
String indexedText = helper.getDocument().getDocumentElement().getTextContent();
indexedText = indexedText.replaceAll("\\r\\n|\\r|\\n", " ");
String normalizedSearchPhrase = searchPhrase.replaceAll("\\*", "");
int index = -1;
if (caseSensitive) {
index = indexedText.indexOf(normalizedSearchPhrase);
} else {
index = indexedText.toLowerCase().indexOf(normalizedSearchPhrase.toLowerCase());
}
if (index != -1) {
int contextLength = (EXCERPT_MAX_LENGTH - normalizedSearchPhrase.length() - 8) / 2;
excerpt = "..." + indexedText.substring(Math.max(index - contextLength, 0), index);
excerpt = excerpt + Ansi.ansi().fg(Ansi.Color.GREEN).toString();
excerpt =
excerpt + indexedText.substring(index, index + normalizedSearchPhrase.length());
excerpt = excerpt + Ansi.ansi().reset().toString();
excerpt =
excerpt
+ indexedText.substring(
index + normalizedSearchPhrase.length(),
Math.min(
indexedText.length(),
index + normalizedSearchPhrase.length() + contextLength))
+ "...";
}
}
}
if (metacard.getModifiedDate() != null) {
modifiedDate =
new DateTime(metacard.getModifiedDate().getTime()).toString(DATETIME_FORMATTER);
}
console.printf(
formatString,
metacard.getId(),
modifiedDate,
title.substring(0, Math.min(title.length(), TITLE_MAX_LENGTH)),
excerpt);
}
return null;
}
@Override
protected Object executeWithSubject() throws Exception {
List<CatalogProvider> providers = getCatalogProviders();
if (providers.isEmpty() || providers.size() < 2) {
console.println("Not enough CatalogProviders installed to migrate");
return null;
}
console.println("The \"FROM\" provider is: " + providers.get(0).getClass().getSimpleName());
CatalogProvider provider = providers.get(1);
console.println("The \"TO\" provider is: " + provider.getClass().getSimpleName());
String answer = getInput("Do you wish to continue? (yes/no)");
if (!"yes".equalsIgnoreCase(answer)) {
console.println();
console.println("Now exiting...");
console.flush();
return null;
}
ingestProvider = new Provider(provider);
framework = getCatalog();
start = System.currentTimeMillis();
final Filter filter =
(cqlFilter != null)
? CQL.toFilter(cqlFilter)
: getFilter(getFilterStartTime(start), start, Metacard.MODIFIED);
QueryImpl query = new QueryImpl(filter);
query.setRequestsTotalResultsCount(true);
query.setPageSize(batchSize);
query.setSortBy(new SortByImpl(Metacard.MODIFIED, SortOrder.DESCENDING));
QueryRequest queryRequest = new QueryRequestImpl(query);
SourceResponse response;
try {
response = framework.query(queryRequest);
} catch (FederationException e) {
printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
return null;
} catch (SourceUnavailableException e) {
printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
return null;
} catch (UnsupportedQueryException e) {
printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
return null;
}
final long totalHits = response.getHits();
final long totalPossible;
if (totalHits == 0) {
console.println("No records were found to replicate.");
return null;
}
// If the maxMetacards is set, restrict the totalPossible to the number of maxMetacards
if (maxMetacards > 0 && maxMetacards <= totalHits) {
totalPossible = maxMetacards;
} else {
totalPossible = totalHits;
}
console.println("Starting migration for " + totalPossible + " Records");
if (multithreaded > 1 && totalPossible > batchSize) {
BlockingQueue<Runnable> blockingQueue = new ArrayBlockingQueue<Runnable>(multithreaded);
RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy();
final ExecutorService executorService =
new ThreadPoolExecutor(
multithreaded,
multithreaded,
0L,
TimeUnit.MILLISECONDS,
blockingQueue,
rejectedExecutionHandler);
console.printf("Running %d threads during replication.%n", multithreaded);
do {
LOGGER.debug("In loop at iteration {}", queryIndex.get());
executorService.submit(
() -> {
int count = queryAndIngest(framework, ingestProvider, queryIndex.get(), filter);
printProgressAndFlush(start, totalPossible, ingestCount.addAndGet(count));
});
} while (queryIndex.addAndGet(batchSize) <= totalPossible);
executorService.shutdown();
while (!executorService.isTerminated()) {
try {
TimeUnit.SECONDS.sleep(1);
} catch (InterruptedException e) {
// ignore
}
}
} else {
do {
int count = queryAndIngest(framework, ingestProvider, queryIndex.get(), filter);
printProgressAndFlush(start, totalPossible, ingestCount.addAndGet(count));
} while (queryIndex.addAndGet(batchSize) <= totalPossible);
}
console.println();
long end = System.currentTimeMillis();
String completed =
String.format(
" %d record(s) replicated; %d record(s) failed; completed in %3.3f seconds.",
ingestCount.get(), failedCount.get(), (end - start) / MS_PER_SECOND);
LOGGER.info("Replication Complete: {}", completed);
console.println(completed);
return null;
}