@Override
protected List<Metacard> query(CatalogFacade framework, int startIndex, Filter filter) {
QueryImpl query = new QueryImpl(filter);
query.setRequestsTotalResultsCount(false);
query.setPageSize(batchSize);
query.setSortBy(new SortByImpl(Metacard.MODIFIED, SortOrder.DESCENDING));
QueryRequest queryRequest = new QueryRequestImpl(query);
query.setStartIndex(startIndex);
SourceResponse response;
try {
LOGGER.debug("Querying with startIndex: {}", startIndex);
response = framework.query(queryRequest);
} catch (UnsupportedQueryException e) {
printErrorMessage(String.format("Received error from Framework: %s%n", e.getMessage()));
return null;
} catch (SourceUnavailableException e) {
printErrorMessage(String.format("Received error from Frameworks: %s%n", e.getMessage()));
return null;
} catch (FederationException e) {
printErrorMessage(String.format("Received error from Frameworks: %s%n", e.getMessage()));
return null;
}
if (response.getProcessingDetails() != null && !response.getProcessingDetails().isEmpty()) {
for (SourceProcessingDetails details : response.getProcessingDetails()) {
LOGGER.debug("Got Issues: {}", details.getWarnings());
}
return null;
}
return response.getResults().stream().map(Result::getMetacard).collect(Collectors.toList());
}
@Test
public void testResultNumReturnedIsZero()
throws WfsException, TransformerConfigurationException, UnsupportedQueryException {
// Setup
final String TITLE = "title";
final String searchPhrase = "*";
final int pageSize = 1;
WfsSource source =
getWfsSource(
ONE_TEXT_PROPERTY_SCHEMA,
MockWfsServer.getFilterCapabilities(),
Wfs20Constants.EPSG_4326_URN,
3,
false,
true,
0);
QueryImpl query =
new QueryImpl(builder.attribute(Metacard.ANY_TEXT).is().like().text(searchPhrase));
query.setPageSize(pageSize);
SortBy sortBy = new SortByImpl(TITLE, SortOrder.DESCENDING);
query.setSortBy(sortBy);
QueryRequestImpl queryReq = new QueryRequestImpl(query);
// Perform test
SourceResponse resp = source.query(queryReq);
assertEquals(3, resp.getResults().size());
}
/**
* Verify that the SortBy is NOT set. In this case, sorting is not supported in the capabilities.
*/
@Test
public void testSortingAscendingSortingNotSupported() throws Exception {
// Setup
final String searchPhrase = "*";
final String mockTemporalFeatureProperty = "myTemporalFeatureProperty";
final String mockFeatureType = "{http://example.com}" + SAMPLE_FEATURE_NAME + 0;
final int pageSize = 1;
// Set ImplementsSorting to FALSE (sorting not supported)
FilterCapabilities mockCapabilitiesSortingNotSupported = MockWfsServer.getFilterCapabilities();
ConformanceType conformance = mockCapabilitiesSortingNotSupported.getConformance();
List<DomainType> domainTypes = conformance.getConstraint();
for (DomainType domainType : domainTypes) {
if (StringUtils.equals(domainType.getName(), "ImplementsSorting")) {
ValueType valueType = new ValueType();
valueType.setValue("FALSE");
domainType.setDefaultValue(valueType);
break;
}
}
WfsSource source =
getWfsSource(
ONE_TEXT_PROPERTY_SCHEMA,
mockCapabilitiesSortingNotSupported,
Wfs20Constants.EPSG_4326_URN,
1,
false,
false,
0);
MetacardMapper mockMetacardMapper = mock(MetacardMapper.class);
when(mockMetacardMapper.getFeatureType()).thenReturn(mockFeatureType);
when(mockMetacardMapper.getSortByTemporalFeatureProperty())
.thenReturn(mockTemporalFeatureProperty);
List<MetacardMapper> mappers = new ArrayList<MetacardMapper>(1);
mappers.add(mockMetacardMapper);
source.setMetacardToFeatureMapper(mappers);
QueryImpl query =
new QueryImpl(builder.attribute(Metacard.ANY_TEXT).is().like().text(searchPhrase));
query.setPageSize(pageSize);
SortBy sortBy = new SortByImpl(Result.TEMPORAL, SortOrder.ASCENDING);
query.setSortBy(sortBy);
// Perform Test
GetFeatureType featureType = source.buildGetFeatureRequest(query);
// Verify
QueryType queryType = (QueryType) featureType.getAbstractQueryExpression().get(0).getValue();
assertFalse(queryType.isSetAbstractSortingClause());
}
/**
* Verify that the SortBy is set with the mapped Feature Property and a DESC sort order. In this
* case, the incoming sort property of TEMPORAL is mapped to myTemporalFeatureProperty.
*
* <p><?xml version="1.0" encoding="UTF-8" standalone="yes"?> <ns5:GetFeature startIndex="1"
* count="1" service="WFS" version="2.0.0" xmlns:ns2="http://www.opengis.net/ows/1.1"
* xmlns="http://www.opengis.net/fes/2.0" xmlns:ns4="http://www.opengis.net/gml"
* xmlns:ns3="http://www.w3.org/1999/xlink" xmlns:ns5="http://www.opengis.net/wfs/2.0"> <ns5:Query
* typeNames="SampleFeature0" handle="SampleFeature0"> <Filter> <PropertyIsLike wildCard="*"
* singleChar="?" escapeChar="!"> <Literal>*</Literal> <ValueReference>title</ValueReference>
* </PropertyIsLike> </Filter> <SortBy> <SortProperty>
* <ValueReference>myTemporalFeatureProperty</ValueReference> <SortOrder>DESC</SortOrder>
* </SortProperty> </SortBy> </ns5:Query> </ns5:GetFeature>
*/
@Test
public void testSortingDescendingSortingSupported() throws Exception {
// Setup
final String searchPhrase = "*";
final String mockTemporalFeatureProperty = "myTemporalFeatureProperty";
final String mockFeatureType = "{http://example.com}" + SAMPLE_FEATURE_NAME + 0;
final int pageSize = 1;
WfsSource source =
getWfsSource(
ONE_TEXT_PROPERTY_SCHEMA,
MockWfsServer.getFilterCapabilities(),
Wfs20Constants.EPSG_4326_URN,
1,
false,
false,
0);
MetacardMapper mockMetacardMapper = mock(MetacardMapper.class);
when(mockMetacardMapper.getFeatureType()).thenReturn(mockFeatureType);
when(mockMetacardMapper.getSortByTemporalFeatureProperty())
.thenReturn(mockTemporalFeatureProperty);
List<MetacardMapper> mappers = new ArrayList<MetacardMapper>(1);
mappers.add(mockMetacardMapper);
source.setMetacardToFeatureMapper(mappers);
QueryImpl query =
new QueryImpl(builder.attribute(Metacard.ANY_TEXT).is().like().text(searchPhrase));
query.setPageSize(pageSize);
SortBy sortBy = new SortByImpl(Result.TEMPORAL, SortOrder.DESCENDING);
query.setSortBy(sortBy);
// Perform Test
GetFeatureType featureType = source.buildGetFeatureRequest(query);
// Verify
QueryType queryType = (QueryType) featureType.getAbstractQueryExpression().get(0).getValue();
JAXBElement<?> abstractSortingClause = queryType.getAbstractSortingClause();
SortByType sortByType = (SortByType) abstractSortingClause.getValue();
assertThat(
sortByType.getSortProperty().get(0).getValueReference(), is(mockTemporalFeatureProperty));
assertThat(
sortByType.getSortProperty().get(0).getSortOrder().name(), is(SortOrderType.DESC.value()));
}
@Test
public void testTypeNameHasNoPrefix() throws WfsException, UnsupportedQueryException {
// Setup
final String TITLE = "title";
final String searchPhrase = "*";
final int pageSize = 1;
WfsSource source =
getWfsSource(
ONE_TEXT_PROPERTY_SCHEMA,
MockWfsServer.getFilterCapabilities(),
Wfs20Constants.EPSG_4326_URN,
3,
false,
false,
3);
QueryImpl query =
new QueryImpl(builder.attribute(Metacard.ANY_TEXT).is().like().text(searchPhrase));
query.setPageSize(pageSize);
SortBy sortBy = new SortByImpl(TITLE, SortOrder.DESCENDING);
query.setSortBy(sortBy);
// Perform test
GetFeatureType featureType = source.buildGetFeatureRequest(query);
// Validate
List<JAXBElement<?>> queryList = featureType.getAbstractQueryExpression();
for (JAXBElement<?> queryType : queryList) {
Object val = queryType.getValue();
QueryType queryTypeVal = (QueryType) val;
assertThat(queryTypeVal.getTypeNames().get(0), containsString("SampleFeature"));
assertThat(queryTypeVal.getTypeNames().get(0), is(not(containsString("Prefix"))));
assertThat(queryTypeVal.getTypeNames().get(0), is(not(containsString(":"))));
}
}
@Override
protected Object executeWithSubject() throws Exception {
List<CatalogProvider> providers = getCatalogProviders();
if (providers.isEmpty() || providers.size() < 2) {
console.println("Not enough CatalogProviders installed to migrate");
return null;
}
console.println("The \"FROM\" provider is: " + providers.get(0).getClass().getSimpleName());
CatalogProvider provider = providers.get(1);
console.println("The \"TO\" provider is: " + provider.getClass().getSimpleName());
String answer = getInput("Do you wish to continue? (yes/no)");
if (!"yes".equalsIgnoreCase(answer)) {
console.println();
console.println("Now exiting...");
console.flush();
return null;
}
ingestProvider = new Provider(provider);
framework = getCatalog();
start = System.currentTimeMillis();
final Filter filter =
(cqlFilter != null)
? CQL.toFilter(cqlFilter)
: getFilter(getFilterStartTime(start), start, Metacard.MODIFIED);
QueryImpl query = new QueryImpl(filter);
query.setRequestsTotalResultsCount(true);
query.setPageSize(batchSize);
query.setSortBy(new SortByImpl(Metacard.MODIFIED, SortOrder.DESCENDING));
QueryRequest queryRequest = new QueryRequestImpl(query);
SourceResponse response;
try {
response = framework.query(queryRequest);
} catch (FederationException e) {
printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
return null;
} catch (SourceUnavailableException e) {
printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
return null;
} catch (UnsupportedQueryException e) {
printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
return null;
}
final long totalHits = response.getHits();
final long totalPossible;
if (totalHits == 0) {
console.println("No records were found to replicate.");
return null;
}
// If the maxMetacards is set, restrict the totalPossible to the number of maxMetacards
if (maxMetacards > 0 && maxMetacards <= totalHits) {
totalPossible = maxMetacards;
} else {
totalPossible = totalHits;
}
console.println("Starting migration for " + totalPossible + " Records");
if (multithreaded > 1 && totalPossible > batchSize) {
BlockingQueue<Runnable> blockingQueue = new ArrayBlockingQueue<Runnable>(multithreaded);
RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy();
final ExecutorService executorService =
new ThreadPoolExecutor(
multithreaded,
multithreaded,
0L,
TimeUnit.MILLISECONDS,
blockingQueue,
rejectedExecutionHandler);
console.printf("Running %d threads during replication.%n", multithreaded);
do {
LOGGER.debug("In loop at iteration {}", queryIndex.get());
executorService.submit(
() -> {
int count = queryAndIngest(framework, ingestProvider, queryIndex.get(), filter);
printProgressAndFlush(start, totalPossible, ingestCount.addAndGet(count));
});
} while (queryIndex.addAndGet(batchSize) <= totalPossible);
executorService.shutdown();
while (!executorService.isTerminated()) {
try {
TimeUnit.SECONDS.sleep(1);
} catch (InterruptedException e) {
// ignore
}
}
} else {
do {
int count = queryAndIngest(framework, ingestProvider, queryIndex.get(), filter);
printProgressAndFlush(start, totalPossible, ingestCount.addAndGet(count));
} while (queryIndex.addAndGet(batchSize) <= totalPossible);
}
console.println();
long end = System.currentTimeMillis();
String completed =
String.format(
" %d record(s) replicated; %d record(s) failed; completed in %3.3f seconds.",
ingestCount.get(), failedCount.get(), (end - start) / MS_PER_SECOND);
LOGGER.info("Replication Complete: {}", completed);
console.println(completed);
return null;
}